calculate hash from store instead of index (#15034)
* calculate hash from store instead of index * restore update hash in abs
This commit is contained in:
parent
d0118a5c42
commit
600ff0d915
|
@ -1,3 +1,5 @@
|
|||
#[macro_use]
|
||||
extern crate log;
|
||||
use clap::{crate_description, crate_name, value_t, App, Arg};
|
||||
use rayon::prelude::*;
|
||||
use solana_measure::measure::Measure;
|
||||
|
@ -51,6 +53,7 @@ fn main() {
|
|||
|
||||
let path = PathBuf::from(env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_owned()))
|
||||
.join("accounts-bench");
|
||||
println!("cleaning file system: {:?}", path);
|
||||
if fs::remove_dir_all(path.clone()).is_err() {
|
||||
println!("Warning: Couldn't remove {:?}", path);
|
||||
}
|
||||
|
@ -84,6 +87,8 @@ fn main() {
|
|||
ancestors.insert(i as u64, i - 1);
|
||||
accounts.add_root(i as u64);
|
||||
}
|
||||
let mut elapsed = vec![0; iterations];
|
||||
let mut elapsed_store = vec![0; iterations];
|
||||
for x in 0..iterations {
|
||||
if clean {
|
||||
let mut time = Measure::start("clean");
|
||||
|
@ -97,13 +102,39 @@ fn main() {
|
|||
} else {
|
||||
let mut pubkeys: Vec<Pubkey> = vec![];
|
||||
let mut time = Measure::start("hash");
|
||||
let hash = accounts
|
||||
let results = accounts
|
||||
.accounts_db
|
||||
.update_accounts_hash(0, &ancestors, true)
|
||||
.0;
|
||||
.update_accounts_hash(0, &ancestors, true);
|
||||
time.stop();
|
||||
println!("hash: {} {}", hash, time);
|
||||
let mut time_store = Measure::start("hash using store");
|
||||
let results_store = accounts.accounts_db.update_accounts_hash_with_index_option(
|
||||
true,
|
||||
false,
|
||||
solana_sdk::clock::Slot::default(),
|
||||
&ancestors,
|
||||
true,
|
||||
);
|
||||
time_store.stop();
|
||||
if results != results_store {
|
||||
error!("results different: \n{:?}\n{:?}", results, results_store);
|
||||
}
|
||||
println!(
|
||||
"hash,{},{},{},{}%",
|
||||
results.0,
|
||||
time,
|
||||
time_store,
|
||||
(time_store.as_us() as f64 / time.as_us() as f64 * 100.0f64) as u32
|
||||
);
|
||||
create_test_accounts(&accounts, &mut pubkeys, 1, 0);
|
||||
elapsed[x] = time.as_us();
|
||||
elapsed_store[x] = time_store.as_us();
|
||||
}
|
||||
}
|
||||
|
||||
for x in elapsed {
|
||||
info!("update_accounts_hash(us),{}", x);
|
||||
}
|
||||
for x in elapsed_store {
|
||||
info!("calculate_accounts_hash_without_index(us),{}", x);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -8,7 +8,9 @@ use crate::{
|
|||
cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES},
|
||||
snapshot_packager_service::PendingSnapshotPackage,
|
||||
};
|
||||
use solana_runtime::snapshot_package::{AccountsPackage, AccountsPackageReceiver};
|
||||
use solana_runtime::snapshot_package::{
|
||||
AccountsPackage, AccountsPackagePre, AccountsPackageReceiver,
|
||||
};
|
||||
use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::{
|
||||
|
@ -49,7 +51,7 @@ impl AccountsHashVerifier {
|
|||
|
||||
match accounts_package_receiver.recv_timeout(Duration::from_secs(1)) {
|
||||
Ok(accounts_package) => {
|
||||
Self::process_accounts_package(
|
||||
Self::process_accounts_package_pre(
|
||||
accounts_package,
|
||||
&cluster_info,
|
||||
&trusted_validators,
|
||||
|
@ -72,6 +74,32 @@ impl AccountsHashVerifier {
|
|||
}
|
||||
}
|
||||
|
||||
fn process_accounts_package_pre(
|
||||
accounts_package: AccountsPackagePre,
|
||||
cluster_info: &ClusterInfo,
|
||||
trusted_validators: &Option<HashSet<Pubkey>>,
|
||||
halt_on_trusted_validator_accounts_hash_mismatch: bool,
|
||||
pending_snapshot_package: &Option<PendingSnapshotPackage>,
|
||||
hashes: &mut Vec<(Slot, Hash)>,
|
||||
exit: &Arc<AtomicBool>,
|
||||
fault_injection_rate_slots: u64,
|
||||
snapshot_interval_slots: u64,
|
||||
) {
|
||||
let accounts_package =
|
||||
solana_runtime::snapshot_utils::process_accounts_package_pre(accounts_package);
|
||||
Self::process_accounts_package(
|
||||
accounts_package,
|
||||
cluster_info,
|
||||
trusted_validators,
|
||||
halt_on_trusted_validator_accounts_hash_mismatch,
|
||||
pending_snapshot_package,
|
||||
hashes,
|
||||
exit,
|
||||
fault_injection_rate_slots,
|
||||
snapshot_interval_slots,
|
||||
);
|
||||
}
|
||||
|
||||
fn process_accounts_package(
|
||||
accounts_package: AccountsPackage,
|
||||
cluster_info: &ClusterInfo,
|
||||
|
@ -83,6 +111,7 @@ impl AccountsHashVerifier {
|
|||
fault_injection_rate_slots: u64,
|
||||
snapshot_interval_slots: u64,
|
||||
) {
|
||||
let hash = accounts_package.hash;
|
||||
if fault_injection_rate_slots != 0
|
||||
&& accounts_package.slot % fault_injection_rate_slots == 0
|
||||
{
|
||||
|
@ -91,10 +120,10 @@ impl AccountsHashVerifier {
|
|||
use solana_sdk::hash::extend_and_hash;
|
||||
warn!("inserting fault at slot: {}", accounts_package.slot);
|
||||
let rand = thread_rng().gen_range(0, 10);
|
||||
let hash = extend_and_hash(&accounts_package.hash, &[rand]);
|
||||
let hash = extend_and_hash(&hash, &[rand]);
|
||||
hashes.push((accounts_package.slot, hash));
|
||||
} else {
|
||||
hashes.push((accounts_package.slot, accounts_package.hash));
|
||||
hashes.push((accounts_package.slot, hash));
|
||||
}
|
||||
|
||||
while hashes.len() > MAX_SNAPSHOT_HASHES {
|
||||
|
|
|
@ -156,7 +156,7 @@ mod tests {
|
|||
|
||||
// Create a packageable snapshot
|
||||
let output_tar_path = snapshot_utils::get_snapshot_archive_path(
|
||||
&snapshot_package_output_path,
|
||||
snapshot_package_output_path,
|
||||
&(42, Hash::default()),
|
||||
ArchiveFormat::TarBzip2,
|
||||
);
|
||||
|
|
|
@ -79,6 +79,7 @@ pub struct TvuConfig {
|
|||
pub repair_validators: Option<HashSet<Pubkey>>,
|
||||
pub accounts_hash_fault_injection_slots: u64,
|
||||
pub accounts_db_caching_enabled: bool,
|
||||
pub test_hash_calculation: bool,
|
||||
}
|
||||
|
||||
impl Tvu {
|
||||
|
@ -274,6 +275,7 @@ impl Tvu {
|
|||
&exit,
|
||||
accounts_background_request_handler,
|
||||
tvu_config.accounts_db_caching_enabled,
|
||||
tvu_config.test_hash_calculation,
|
||||
);
|
||||
|
||||
Tvu {
|
||||
|
|
|
@ -121,6 +121,7 @@ pub struct ValidatorConfig {
|
|||
pub account_indexes: HashSet<AccountIndex>,
|
||||
pub accounts_db_caching_enabled: bool,
|
||||
pub warp_slot: Option<Slot>,
|
||||
pub accounts_db_test_hash_calculation: bool,
|
||||
}
|
||||
|
||||
impl Default for ValidatorConfig {
|
||||
|
@ -168,6 +169,7 @@ impl Default for ValidatorConfig {
|
|||
account_indexes: HashSet::new(),
|
||||
accounts_db_caching_enabled: false,
|
||||
warp_slot: None,
|
||||
accounts_db_test_hash_calculation: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -641,6 +643,7 @@ impl Validator {
|
|||
repair_validators: config.repair_validators.clone(),
|
||||
accounts_hash_fault_injection_slots: config.accounts_hash_fault_injection_slots,
|
||||
accounts_db_caching_enabled: config.accounts_db_caching_enabled,
|
||||
test_hash_calculation: config.accounts_db_test_hash_calculation,
|
||||
},
|
||||
);
|
||||
|
||||
|
|
|
@ -153,7 +153,7 @@ mod tests {
|
|||
.unwrap()
|
||||
.snapshot_path,
|
||||
snapshot_utils::get_snapshot_archive_path(
|
||||
snapshot_package_output_path,
|
||||
snapshot_package_output_path.to_path_buf(),
|
||||
&(old_last_bank.slot(), old_last_bank.get_accounts_hash()),
|
||||
ArchiveFormat::TarBzip2,
|
||||
),
|
||||
|
@ -218,7 +218,8 @@ mod tests {
|
|||
if slot % set_root_interval == 0 || slot == last_slot - 1 {
|
||||
// set_root should send a snapshot request
|
||||
bank_forks.set_root(bank.slot(), &request_sender, None);
|
||||
snapshot_request_handler.handle_snapshot_requests(false);
|
||||
bank.update_accounts_hash();
|
||||
snapshot_request_handler.handle_snapshot_requests(false, false);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -238,8 +239,10 @@ mod tests {
|
|||
last_bank.get_snapshot_storages(),
|
||||
ArchiveFormat::TarBzip2,
|
||||
snapshot_version,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let snapshot_package = snapshot_utils::process_accounts_package_pre(snapshot_package);
|
||||
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
|
||||
|
||||
// Restore bank from snapshot
|
||||
|
@ -358,6 +361,7 @@ mod tests {
|
|||
&snapshot_package_output_path,
|
||||
snapshot_config.snapshot_version,
|
||||
&snapshot_config.archive_format,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
@ -383,7 +387,7 @@ mod tests {
|
|||
fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
|
||||
|
||||
saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
|
||||
snapshot_package_output_path,
|
||||
snapshot_package_output_path.to_path_buf(),
|
||||
&(slot, accounts_hash),
|
||||
ArchiveFormat::TarBzip2,
|
||||
));
|
||||
|
@ -425,6 +429,10 @@ mod tests {
|
|||
snapshot_package = new_snapshot_package;
|
||||
}
|
||||
|
||||
let snapshot_package =
|
||||
solana_runtime::snapshot_utils::process_accounts_package_pre(
|
||||
snapshot_package,
|
||||
);
|
||||
*pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
|
||||
}
|
||||
|
||||
|
|
|
@ -180,7 +180,7 @@ pub fn download_snapshot(
|
|||
ArchiveFormat::TarBzip2,
|
||||
] {
|
||||
let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
|
||||
ledger_path,
|
||||
ledger_path.to_path_buf(),
|
||||
&desired_snapshot_hash,
|
||||
*compression,
|
||||
);
|
||||
|
|
|
@ -1052,7 +1052,10 @@ fn test_snapshot_download() {
|
|||
|
||||
trace!("found: {:?}", archive_filename);
|
||||
let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
|
||||
&validator_snapshot_test_config.snapshot_output_path,
|
||||
validator_snapshot_test_config
|
||||
.snapshot_output_path
|
||||
.path()
|
||||
.to_path_buf(),
|
||||
&archive_snapshot_hash,
|
||||
ArchiveFormat::TarBzip2,
|
||||
);
|
||||
|
@ -1122,7 +1125,10 @@ fn test_snapshot_restart_tower() {
|
|||
|
||||
// Copy archive to validator's snapshot output directory
|
||||
let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
|
||||
&validator_snapshot_test_config.snapshot_output_path,
|
||||
validator_snapshot_test_config
|
||||
.snapshot_output_path
|
||||
.path()
|
||||
.to_path_buf(),
|
||||
&archive_snapshot_hash,
|
||||
ArchiveFormat::TarBzip2,
|
||||
);
|
||||
|
@ -1187,7 +1193,10 @@ fn test_snapshots_blockstore_floor() {
|
|||
|
||||
// Copy archive to validator's snapshot output directory
|
||||
let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
|
||||
&validator_snapshot_test_config.snapshot_output_path,
|
||||
validator_snapshot_test_config
|
||||
.snapshot_output_path
|
||||
.path()
|
||||
.to_path_buf(),
|
||||
&(archive_slot, archive_hash),
|
||||
ArchiveFormat::TarBzip2,
|
||||
);
|
||||
|
|
|
@ -14,7 +14,7 @@ blake3 = "0.3.6"
|
|||
bv = { version = "0.11.1", features = ["serde"] }
|
||||
byteorder = "1.3.4"
|
||||
bzip2 = "0.3.3"
|
||||
dashmap = { version = "4.0.2", features = ["rayon"] }
|
||||
dashmap = { version = "4.0.2", features = ["rayon", "raw-api"] }
|
||||
crossbeam-channel = "0.4"
|
||||
dir-diff = "0.3.2"
|
||||
flate2 = "1.0.14"
|
||||
|
|
|
@ -77,7 +77,11 @@ pub struct SnapshotRequestHandler {
|
|||
|
||||
impl SnapshotRequestHandler {
|
||||
// Returns the latest requested snapshot slot, if one exists
|
||||
pub fn handle_snapshot_requests(&self, accounts_db_caching_enabled: bool) -> Option<u64> {
|
||||
pub fn handle_snapshot_requests(
|
||||
&self,
|
||||
accounts_db_caching_enabled: bool,
|
||||
test_hash_calculation: bool,
|
||||
) -> Option<u64> {
|
||||
self.snapshot_request_receiver
|
||||
.try_iter()
|
||||
.last()
|
||||
|
@ -87,10 +91,6 @@ impl SnapshotRequestHandler {
|
|||
status_cache_slot_deltas,
|
||||
} = snapshot_request;
|
||||
|
||||
let mut hash_time = Measure::start("hash_time");
|
||||
snapshot_root_bank.update_accounts_hash();
|
||||
hash_time.stop();
|
||||
|
||||
let mut shrink_time = Measure::start("shrink_time");
|
||||
if !accounts_db_caching_enabled {
|
||||
snapshot_root_bank
|
||||
|
@ -120,6 +120,15 @@ impl SnapshotRequestHandler {
|
|||
}
|
||||
flush_accounts_cache_time.stop();
|
||||
|
||||
let mut hash_time = Measure::start("hash_time");
|
||||
let mut hash_for_testing = None;
|
||||
snapshot_root_bank
|
||||
.update_accounts_hash_with_index_option(true, test_hash_calculation);
|
||||
if test_hash_calculation {
|
||||
hash_for_testing = Some(snapshot_root_bank.get_accounts_hash());
|
||||
}
|
||||
hash_time.stop();
|
||||
|
||||
let mut clean_time = Measure::start("clean_time");
|
||||
// Don't clean the slot we're snapshotting because it may have zero-lamport
|
||||
// accounts that were included in the bank delta hash when the bank was frozen,
|
||||
|
@ -144,6 +153,7 @@ impl SnapshotRequestHandler {
|
|||
&self.snapshot_config.snapshot_package_output_path,
|
||||
self.snapshot_config.snapshot_version,
|
||||
&self.snapshot_config.archive_format,
|
||||
hash_for_testing,
|
||||
);
|
||||
if r.is_err() {
|
||||
warn!(
|
||||
|
@ -216,11 +226,16 @@ pub struct ABSRequestHandler {
|
|||
|
||||
impl ABSRequestHandler {
|
||||
// Returns the latest requested snapshot block height, if one exists
|
||||
pub fn handle_snapshot_requests(&self, accounts_db_caching_enabled: bool) -> Option<u64> {
|
||||
pub fn handle_snapshot_requests(
|
||||
&self,
|
||||
accounts_db_caching_enabled: bool,
|
||||
test_hash_calculation: bool,
|
||||
) -> Option<u64> {
|
||||
self.snapshot_request_handler
|
||||
.as_ref()
|
||||
.and_then(|snapshot_request_handler| {
|
||||
snapshot_request_handler.handle_snapshot_requests(accounts_db_caching_enabled)
|
||||
snapshot_request_handler
|
||||
.handle_snapshot_requests(accounts_db_caching_enabled, test_hash_calculation)
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -245,6 +260,7 @@ impl AccountsBackgroundService {
|
|||
exit: &Arc<AtomicBool>,
|
||||
request_handler: ABSRequestHandler,
|
||||
accounts_db_caching_enabled: bool,
|
||||
test_hash_calculation: bool,
|
||||
) -> Self {
|
||||
info!("AccountsBackgroundService active");
|
||||
let exit = exit.clone();
|
||||
|
@ -287,8 +303,8 @@ impl AccountsBackgroundService {
|
|||
// request for `N` to the snapshot request channel before setting a root `R > N`, and
|
||||
// snapshot_request_handler.handle_requests() will always look for the latest
|
||||
// available snapshot in the channel.
|
||||
let snapshot_block_height =
|
||||
request_handler.handle_snapshot_requests(accounts_db_caching_enabled);
|
||||
let snapshot_block_height = request_handler
|
||||
.handle_snapshot_requests(accounts_db_caching_enabled, test_hash_calculation);
|
||||
if accounts_db_caching_enabled {
|
||||
// Note that the flush will do an internal clean of the
|
||||
// cache up to bank.slot(), so should be safe as long
|
||||
|
|
|
@ -167,6 +167,27 @@ type ReclaimResult = (AccountSlots, AppendVecOffsets);
|
|||
type StorageFinder<'a> = Box<dyn Fn(Slot, usize) -> Arc<AccountStorageEntry> + 'a>;
|
||||
type ShrinkCandidates = HashMap<Slot, HashMap<AppendVecId, Arc<AccountStorageEntry>>>;
|
||||
|
||||
#[derive(Default, Debug, PartialEq, Clone)]
|
||||
struct CalculateHashIntermediate {
|
||||
pub version: u64,
|
||||
pub hash: Hash,
|
||||
pub lamports: u64,
|
||||
pub raw_lamports: u64,
|
||||
pub slot: Slot,
|
||||
}
|
||||
|
||||
impl CalculateHashIntermediate {
|
||||
pub fn new(version: u64, hash: Hash, lamports: u64, raw_lamports: u64, slot: Slot) -> Self {
|
||||
Self {
|
||||
version,
|
||||
hash,
|
||||
lamports,
|
||||
raw_lamports,
|
||||
slot,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait Versioned {
|
||||
fn version(&self) -> u64;
|
||||
}
|
||||
|
@ -177,6 +198,12 @@ impl Versioned for (u64, Hash) {
|
|||
}
|
||||
}
|
||||
|
||||
impl Versioned for CalculateHashIntermediate {
|
||||
fn version(&self) -> u64 {
|
||||
self.version
|
||||
}
|
||||
}
|
||||
|
||||
impl Versioned for (u64, AccountInfo) {
|
||||
fn version(&self) -> u64 {
|
||||
self.0
|
||||
|
@ -286,6 +313,16 @@ impl<'a> LoadedAccount<'a> {
|
|||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn lamports(&self) -> u64 {
|
||||
match self {
|
||||
LoadedAccount::Stored(stored_account_meta) => stored_account_meta.account_meta.lamports,
|
||||
LoadedAccount::Cached((_, cached_account)) => match cached_account {
|
||||
Cow::Owned(cached_account) => cached_account.account.lamports,
|
||||
Cow::Borrowed(cached_account) => cached_account.account.lamports,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Debug)]
|
||||
|
@ -3402,12 +3439,13 @@ impl AccountsDB {
|
|||
slot: Slot,
|
||||
debug: bool,
|
||||
) -> Hash {
|
||||
let (hash, ..) = Self::accumulate_account_hashes_and_capitalization(hashes, slot, debug).0;
|
||||
let ((hash, ..), ..) =
|
||||
Self::accumulate_account_hashes_and_capitalization(hashes, slot, debug);
|
||||
hash
|
||||
}
|
||||
|
||||
fn sort_hashes_by_pubkey(hashes: &mut Vec<(Pubkey, Hash, u64)>) {
|
||||
hashes.par_sort_by(|a, b| a.0.cmp(&b.0));
|
||||
hashes.par_sort_unstable_by(|a, b| a.0.cmp(&b.0));
|
||||
}
|
||||
|
||||
fn accumulate_account_hashes_and_capitalization(
|
||||
|
@ -3574,15 +3612,236 @@ impl AccountsDB {
|
|||
ancestors: &Ancestors,
|
||||
simple_capitalization_enabled: bool,
|
||||
) -> (Hash, u64) {
|
||||
let (hash, total_lamports) = self
|
||||
.calculate_accounts_hash(slot, ancestors, false, simple_capitalization_enabled)
|
||||
.unwrap();
|
||||
self.update_accounts_hash_with_index_option(
|
||||
false,
|
||||
false,
|
||||
slot,
|
||||
ancestors,
|
||||
simple_capitalization_enabled,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn update_accounts_hash_test(
|
||||
&self,
|
||||
slot: Slot,
|
||||
ancestors: &Ancestors,
|
||||
simple_capitalization_enabled: bool,
|
||||
) -> (Hash, u64) {
|
||||
self.update_accounts_hash_with_index_option(
|
||||
false,
|
||||
true,
|
||||
slot,
|
||||
ancestors,
|
||||
simple_capitalization_enabled,
|
||||
)
|
||||
}
|
||||
|
||||
/// Scan through all the account storage in parallel
|
||||
fn scan_account_storage_no_bank<F, B>(
|
||||
snapshot_storages: SnapshotStorages,
|
||||
scan_func: F,
|
||||
) -> Vec<B>
|
||||
where
|
||||
F: Fn(LoadedAccount, AppendVecId, &mut B, Slot) + Send + Sync,
|
||||
B: Send + Default,
|
||||
{
|
||||
snapshot_storages
|
||||
.into_par_iter()
|
||||
.flatten()
|
||||
.map(|storage| {
|
||||
let accounts = storage.accounts.accounts(0);
|
||||
let mut retval = B::default();
|
||||
accounts.into_iter().for_each(|stored_account| {
|
||||
scan_func(
|
||||
LoadedAccount::Stored(stored_account),
|
||||
storage.append_vec_id(),
|
||||
&mut retval,
|
||||
storage.slot(),
|
||||
)
|
||||
});
|
||||
retval
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn remove_zero_balance_accounts(
|
||||
account_maps: DashMap<Pubkey, CalculateHashIntermediate>,
|
||||
) -> Vec<(Pubkey, Hash, u64)> {
|
||||
type ShardType = dashmap::lock::RwLock<
|
||||
std::collections::HashMap<
|
||||
solana_sdk::pubkey::Pubkey,
|
||||
dashmap::SharedValue<CalculateHashIntermediate>,
|
||||
>,
|
||||
>;
|
||||
let shards: &[ShardType] = account_maps.shards();
|
||||
|
||||
let hashes: Vec<_> = shards
|
||||
.par_iter()
|
||||
.map(|x| {
|
||||
let a: dashmap::lock::RwLockReadGuard<HashMap<_, _>> = x.read();
|
||||
let res: Vec<_> = a
|
||||
.iter()
|
||||
.filter_map(|inp| {
|
||||
let (pubkey, sv) = inp;
|
||||
let item = sv.get();
|
||||
if item.raw_lamports != 0 {
|
||||
Some((*pubkey, item.hash, item.lamports))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
res
|
||||
})
|
||||
.flatten()
|
||||
.collect();
|
||||
hashes
|
||||
}
|
||||
|
||||
fn rest_of_hash_calculation(
|
||||
accounts: (DashMap<Pubkey, CalculateHashIntermediate>, Measure),
|
||||
) -> (Hash, u64) {
|
||||
let (account_maps, time_scan) = accounts;
|
||||
|
||||
let mut zeros = Measure::start("eliminate zeros");
|
||||
let hashes = Self::remove_zero_balance_accounts(account_maps);
|
||||
zeros.stop();
|
||||
let hash_total = hashes.len();
|
||||
let (ret, (sort_time, hash_time)) =
|
||||
Self::accumulate_account_hashes_and_capitalization(hashes, Slot::default(), false);
|
||||
datapoint_info!(
|
||||
"calculate_accounts_hash_without_index",
|
||||
("accounts_scan", time_scan.as_us(), i64),
|
||||
("eliminate_zeros", zeros.as_us(), i64),
|
||||
("hash", hash_time.as_us(), i64),
|
||||
("sort", sort_time.as_us(), i64),
|
||||
("hash_total", hash_total, i64),
|
||||
);
|
||||
|
||||
ret
|
||||
}
|
||||
|
||||
fn calculate_accounts_hash_helper(
|
||||
&self,
|
||||
do_not_use_index: bool,
|
||||
slot: Slot,
|
||||
ancestors: &Ancestors,
|
||||
simple_capitalization_enabled: bool,
|
||||
) -> (Hash, u64) {
|
||||
if do_not_use_index {
|
||||
let combined_maps = self.get_snapshot_storages(slot);
|
||||
|
||||
Self::calculate_accounts_hash_without_index(
|
||||
combined_maps,
|
||||
simple_capitalization_enabled,
|
||||
)
|
||||
} else {
|
||||
self.calculate_accounts_hash(slot, ancestors, false, simple_capitalization_enabled)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_accounts_hash_with_index_option(
|
||||
&self,
|
||||
do_not_use_index: bool,
|
||||
debug_verify: bool,
|
||||
slot: Slot,
|
||||
ancestors: &Ancestors,
|
||||
simple_capitalization_enabled: bool,
|
||||
) -> (Hash, u64) {
|
||||
let (hash, total_lamports) = self.calculate_accounts_hash_helper(
|
||||
do_not_use_index,
|
||||
slot,
|
||||
ancestors,
|
||||
simple_capitalization_enabled,
|
||||
);
|
||||
if debug_verify {
|
||||
// calculate the other way (store or non-store) and verify results match.
|
||||
let (hash_other, total_lamports_other) = self.calculate_accounts_hash_helper(
|
||||
!do_not_use_index,
|
||||
slot,
|
||||
ancestors,
|
||||
simple_capitalization_enabled,
|
||||
);
|
||||
|
||||
assert_eq!(hash, hash_other);
|
||||
assert_eq!(total_lamports, total_lamports_other);
|
||||
}
|
||||
let mut bank_hashes = self.bank_hashes.write().unwrap();
|
||||
let mut bank_hash_info = bank_hashes.get_mut(&slot).unwrap();
|
||||
bank_hash_info.snapshot_hash = hash;
|
||||
(hash, total_lamports)
|
||||
}
|
||||
|
||||
fn handle_one_loaded_account(
|
||||
key: &Pubkey,
|
||||
found_item: CalculateHashIntermediate,
|
||||
map: &DashMap<Pubkey, CalculateHashIntermediate>,
|
||||
) {
|
||||
match map.entry(*key) {
|
||||
Occupied(mut dest_item) => {
|
||||
let contents = dest_item.get();
|
||||
if contents.slot < found_item.slot
|
||||
|| (contents.slot == found_item.slot
|
||||
&& contents.version() <= found_item.version())
|
||||
{
|
||||
// replace the item
|
||||
dest_item.insert(found_item);
|
||||
}
|
||||
}
|
||||
Vacant(v) => {
|
||||
v.insert(found_item);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
fn scan_snapshot_stores(
|
||||
storage: SnapshotStorages,
|
||||
simple_capitalization_enabled: bool,
|
||||
) -> (DashMap<Pubkey, CalculateHashIntermediate>, Measure) {
|
||||
let map: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
|
||||
let mut time = Measure::start("scan all accounts");
|
||||
Self::scan_account_storage_no_bank(
|
||||
storage,
|
||||
|loaded_account: LoadedAccount,
|
||||
_store_id: AppendVecId,
|
||||
_accum: &mut Vec<(Pubkey, CalculateHashIntermediate)>,
|
||||
slot: Slot| {
|
||||
let version = loaded_account.write_version();
|
||||
let raw_lamports = loaded_account.lamports();
|
||||
let balance = Self::account_balance_for_capitalization(
|
||||
raw_lamports,
|
||||
loaded_account.owner(),
|
||||
loaded_account.executable(),
|
||||
simple_capitalization_enabled,
|
||||
);
|
||||
|
||||
let source_item = CalculateHashIntermediate::new(
|
||||
version,
|
||||
*loaded_account.loaded_hash(),
|
||||
balance,
|
||||
raw_lamports,
|
||||
slot,
|
||||
);
|
||||
Self::handle_one_loaded_account(loaded_account.pubkey(), source_item, &map);
|
||||
},
|
||||
);
|
||||
time.stop();
|
||||
|
||||
(map, time)
|
||||
}
|
||||
|
||||
// modeled after get_accounts_delta_hash
|
||||
// intended to be faster than calculate_accounts_hash
|
||||
pub fn calculate_accounts_hash_without_index(
|
||||
storages: SnapshotStorages,
|
||||
simple_capitalization_enabled: bool,
|
||||
) -> (Hash, u64) {
|
||||
let result = Self::scan_snapshot_stores(storages, simple_capitalization_enabled);
|
||||
|
||||
Self::rest_of_hash_calculation(result)
|
||||
}
|
||||
|
||||
pub fn verify_bank_hash_and_lamports(
|
||||
&self,
|
||||
slot: Slot,
|
||||
|
@ -4788,6 +5047,163 @@ pub mod tests {
|
|||
ancestors
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accountsdb_rest_of_hash_calculation() {
|
||||
solana_logger::setup();
|
||||
|
||||
let key = Pubkey::new(&[11u8; 32]);
|
||||
let account_maps: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
|
||||
let hash = Hash::new(&[1u8; 32]);
|
||||
let val = CalculateHashIntermediate::new(0, hash, 88, 490, Slot::default());
|
||||
account_maps.insert(key, val);
|
||||
|
||||
// 2nd key - zero lamports, so will be removed
|
||||
let key = Pubkey::new(&[12u8; 32]);
|
||||
let hash = Hash::new(&[2u8; 32]);
|
||||
let val = CalculateHashIntermediate::new(0, hash, 1, 0, Slot::default());
|
||||
account_maps.insert(key, val);
|
||||
|
||||
let result =
|
||||
AccountsDB::rest_of_hash_calculation((account_maps.clone(), Measure::start("")));
|
||||
let expected_hash = Hash::from_str("8j9ARGFv4W2GfML7d3sVJK2MePwrikqYnu6yqer28cCa").unwrap();
|
||||
assert_eq!(result, (expected_hash, 88));
|
||||
|
||||
// 3rd key - with pubkey value before 1st key so it will be sorted first
|
||||
let key = Pubkey::new(&[10u8; 32]);
|
||||
let hash = Hash::new(&[2u8; 32]);
|
||||
let val = CalculateHashIntermediate::new(0, hash, 20, 20, Slot::default());
|
||||
account_maps.insert(key, val);
|
||||
|
||||
let result = AccountsDB::rest_of_hash_calculation((account_maps, Measure::start("")));
|
||||
let expected_hash = Hash::from_str("EHv9C5vX7xQjjMpsJMzudnDTzoTSRwYkqLzY8tVMihGj").unwrap();
|
||||
assert_eq!(result, (expected_hash, 108));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accountsdb_handle_one_loaded_account() {
|
||||
solana_logger::setup();
|
||||
|
||||
let account_maps: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
|
||||
let key = Pubkey::new_unique();
|
||||
let hash = Hash::new_unique();
|
||||
let val = CalculateHashIntermediate::new(1, hash, 1, 2, 1);
|
||||
|
||||
AccountsDB::handle_one_loaded_account(&key, val.clone(), &account_maps);
|
||||
assert_eq!(*account_maps.get(&key).unwrap(), val);
|
||||
|
||||
// slot same, version <
|
||||
let hash2 = Hash::new_unique();
|
||||
let val2 = CalculateHashIntermediate::new(0, hash2, 4, 5, 1);
|
||||
AccountsDB::handle_one_loaded_account(&key, val2, &account_maps);
|
||||
assert_eq!(*account_maps.get(&key).unwrap(), val);
|
||||
|
||||
// slot same, vers =
|
||||
let hash3 = Hash::new_unique();
|
||||
let val3 = CalculateHashIntermediate::new(1, hash3, 2, 3, 1);
|
||||
AccountsDB::handle_one_loaded_account(&key, val3.clone(), &account_maps);
|
||||
assert_eq!(*account_maps.get(&key).unwrap(), val3);
|
||||
|
||||
// slot same, vers >
|
||||
let hash4 = Hash::new_unique();
|
||||
let val4 = CalculateHashIntermediate::new(2, hash4, 6, 7, 1);
|
||||
AccountsDB::handle_one_loaded_account(&key, val4.clone(), &account_maps);
|
||||
assert_eq!(*account_maps.get(&key).unwrap(), val4);
|
||||
|
||||
// slot >, version <
|
||||
let hash5 = Hash::new_unique();
|
||||
let val5 = CalculateHashIntermediate::new(0, hash5, 8, 9, 2);
|
||||
AccountsDB::handle_one_loaded_account(&key, val5.clone(), &account_maps);
|
||||
assert_eq!(*account_maps.get(&key).unwrap(), val5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accountsdb_remove_zero_balance_accounts() {
|
||||
solana_logger::setup();
|
||||
|
||||
let key = Pubkey::new_unique();
|
||||
let hash = Hash::new_unique();
|
||||
let account_maps: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
|
||||
let val = CalculateHashIntermediate::new(0, hash, 1, 2, Slot::default());
|
||||
account_maps.insert(key, val.clone());
|
||||
|
||||
let result = AccountsDB::remove_zero_balance_accounts(account_maps);
|
||||
assert_eq!(result, vec![(key, val.hash, val.lamports)]);
|
||||
|
||||
// zero original lamports
|
||||
let account_maps: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
|
||||
let val = CalculateHashIntermediate::new(0, hash, 1, 0, Slot::default());
|
||||
account_maps.insert(key, val);
|
||||
|
||||
let result = AccountsDB::remove_zero_balance_accounts(account_maps);
|
||||
assert_eq!(result, vec![]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accountsdb_calculate_accounts_hash_without_index_simple() {
|
||||
solana_logger::setup();
|
||||
|
||||
let (storages, _size, _slot_expected) = sample_storage();
|
||||
let result = AccountsDB::calculate_accounts_hash_without_index(storages, true);
|
||||
let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap();
|
||||
assert_eq!(result, (expected_hash, 0));
|
||||
}
|
||||
|
||||
fn sample_storage() -> (SnapshotStorages, usize, Slot) {
|
||||
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
|
||||
let slot_expected: Slot = 0;
|
||||
let size: usize = 123;
|
||||
let data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
|
||||
|
||||
let arc = Arc::new(data);
|
||||
let storages = vec![vec![arc]];
|
||||
(storages, size, slot_expected)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accountsdb_scan_account_storage_no_bank() {
|
||||
solana_logger::setup();
|
||||
|
||||
let expected = 1;
|
||||
let tf = crate::append_vec::test_utils::get_append_vec_path(
|
||||
"test_accountsdb_scan_account_storage_no_bank",
|
||||
);
|
||||
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
|
||||
let slot_expected: Slot = 0;
|
||||
let size: usize = 123;
|
||||
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
|
||||
let av = AppendVec::new(&tf.path, true, 1024 * 1024);
|
||||
data.accounts = av;
|
||||
|
||||
let arc = Arc::new(data);
|
||||
let storages = vec![vec![arc]];
|
||||
let pubkey = solana_sdk::pubkey::new_rand();
|
||||
let acc = Account::new(1, 48, &Account::default().owner);
|
||||
let sm = StoredMeta {
|
||||
data_len: 1,
|
||||
pubkey,
|
||||
write_version: 1,
|
||||
};
|
||||
storages[0][0]
|
||||
.accounts
|
||||
.append_accounts(&[(sm, &acc)], &[Hash::default()]);
|
||||
|
||||
let calls = AtomicU64::new(0);
|
||||
let result = AccountsDB::scan_account_storage_no_bank(
|
||||
storages,
|
||||
|loaded_account: LoadedAccount,
|
||||
_store_id: AppendVecId,
|
||||
accum: &mut Vec<u64>,
|
||||
slot: Slot| {
|
||||
calls.fetch_add(1, Ordering::Relaxed);
|
||||
assert_eq!(loaded_account.pubkey(), &pubkey);
|
||||
assert_eq!(slot_expected, slot);
|
||||
accum.push(expected);
|
||||
},
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(result, vec![vec![expected]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accountsdb_compute_merkle_root_and_capitalization() {
|
||||
solana_logger::setup();
|
||||
|
@ -4841,10 +5257,18 @@ pub mod tests {
|
|||
} else {
|
||||
result = AccountsDB::accumulate_account_hashes_and_capitalization(
|
||||
input.clone(),
|
||||
0,
|
||||
Slot::default(),
|
||||
false,
|
||||
)
|
||||
.0;
|
||||
assert_eq!(
|
||||
AccountsDB::accumulate_account_hashes(
|
||||
input.clone(),
|
||||
Slot::default(),
|
||||
false
|
||||
),
|
||||
result.0
|
||||
);
|
||||
AccountsDB::sort_hashes_by_pubkey(&mut input);
|
||||
}
|
||||
let mut expected = 0;
|
||||
|
@ -6051,12 +6475,12 @@ pub mod tests {
|
|||
|
||||
let ancestors = linear_ancestors(current_slot);
|
||||
info!("ancestors: {:?}", ancestors);
|
||||
let hash = accounts.update_accounts_hash(current_slot, &ancestors, true);
|
||||
let hash = accounts.update_accounts_hash_test(current_slot, &ancestors, true);
|
||||
|
||||
accounts.clean_accounts(None);
|
||||
|
||||
assert_eq!(
|
||||
accounts.update_accounts_hash(current_slot, &ancestors, true),
|
||||
accounts.update_accounts_hash_test(current_slot, &ancestors, true),
|
||||
hash
|
||||
);
|
||||
|
||||
|
@ -6559,7 +6983,7 @@ pub mod tests {
|
|||
|
||||
db.store_uncached(some_slot, &[(&key, &account)]);
|
||||
db.add_root(some_slot);
|
||||
db.update_accounts_hash(some_slot, &ancestors, true);
|
||||
db.update_accounts_hash_test(some_slot, &ancestors, true);
|
||||
assert_matches!(
|
||||
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, true),
|
||||
Ok(_)
|
||||
|
@ -6601,7 +7025,7 @@ pub mod tests {
|
|||
|
||||
db.store_uncached(some_slot, &[(&key, &account)]);
|
||||
db.add_root(some_slot);
|
||||
db.update_accounts_hash(some_slot, &ancestors, true);
|
||||
db.update_accounts_hash_test(some_slot, &ancestors, true);
|
||||
assert_matches!(
|
||||
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, true),
|
||||
Ok(_)
|
||||
|
@ -6615,7 +7039,7 @@ pub mod tests {
|
|||
&solana_sdk::native_loader::create_loadable_account("foo", 1),
|
||||
)],
|
||||
);
|
||||
db.update_accounts_hash(some_slot, &ancestors, true);
|
||||
db.update_accounts_hash_test(some_slot, &ancestors, true);
|
||||
assert_matches!(
|
||||
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, false),
|
||||
Ok(_)
|
||||
|
@ -6644,7 +7068,7 @@ pub mod tests {
|
|||
.unwrap()
|
||||
.insert(some_slot, BankHashInfo::default());
|
||||
db.add_root(some_slot);
|
||||
db.update_accounts_hash(some_slot, &ancestors, true);
|
||||
db.update_accounts_hash_test(some_slot, &ancestors, true);
|
||||
assert_matches!(
|
||||
db.verify_bank_hash_and_lamports(some_slot, &ancestors, 0, true),
|
||||
Ok(_)
|
||||
|
|
|
@ -4299,16 +4299,30 @@ impl Bank {
|
|||
self.rc.accounts.accounts_db.get_accounts_hash(self.slot)
|
||||
}
|
||||
|
||||
pub fn update_accounts_hash(&self) -> Hash {
|
||||
let (hash, total_lamports) = self.rc.accounts.accounts_db.update_accounts_hash(
|
||||
self.slot(),
|
||||
&self.ancestors,
|
||||
self.simple_capitalization_enabled(),
|
||||
);
|
||||
pub fn update_accounts_hash_with_index_option(
|
||||
&self,
|
||||
do_not_use_index: bool,
|
||||
debug_verify: bool,
|
||||
) -> Hash {
|
||||
let (hash, total_lamports) = self
|
||||
.rc
|
||||
.accounts
|
||||
.accounts_db
|
||||
.update_accounts_hash_with_index_option(
|
||||
do_not_use_index,
|
||||
debug_verify,
|
||||
self.slot(),
|
||||
&self.ancestors,
|
||||
self.simple_capitalization_enabled(),
|
||||
);
|
||||
assert_eq!(total_lamports, self.capitalization());
|
||||
hash
|
||||
}
|
||||
|
||||
pub fn update_accounts_hash(&self) -> Hash {
|
||||
self.update_accounts_hash_with_index_option(false, false)
|
||||
}
|
||||
|
||||
/// A snapshot bank should be purged of 0 lamport accounts which are not part of the hash
|
||||
/// calculation and could shield other real accounts.
|
||||
pub fn verify_snapshot_bank(&self) -> bool {
|
||||
|
@ -4942,7 +4956,10 @@ fn is_simple_vote_transaction(transaction: &Transaction) -> bool {
|
|||
if program_pubkey == solana_vote_program::id() {
|
||||
if let Ok(vote_instruction) = limited_deserialize::<VoteInstruction>(&instruction.data)
|
||||
{
|
||||
return matches!(vote_instruction, VoteInstruction::Vote(_) | VoteInstruction::VoteSwitch(_, _));
|
||||
return matches!(
|
||||
vote_instruction,
|
||||
VoteInstruction::Vote(_) | VoteInstruction::VoteSwitch(_, _)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -7955,6 +7972,7 @@ pub(crate) mod tests {
|
|||
|
||||
#[test]
|
||||
fn test_verify_snapshot_bank() {
|
||||
solana_logger::setup();
|
||||
let pubkey = solana_sdk::pubkey::new_rand();
|
||||
let (genesis_config, mint_keypair) = create_genesis_config(2_000);
|
||||
let bank = Bank::new(&genesis_config);
|
||||
|
|
|
@ -9,11 +9,59 @@ use std::{
|
|||
};
|
||||
use tempfile::TempDir;
|
||||
|
||||
pub type AccountsPackageSender = Sender<AccountsPackage>;
|
||||
pub type AccountsPackageReceiver = Receiver<AccountsPackage>;
|
||||
pub type AccountsPackageSendError = SendError<AccountsPackage>;
|
||||
pub type AccountsPackageSender = Sender<AccountsPackagePre>;
|
||||
pub type AccountsPackageReceiver = Receiver<AccountsPackagePre>;
|
||||
pub type AccountsPackageSendError = SendError<AccountsPackagePre>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct AccountsPackagePre {
|
||||
pub slot: Slot,
|
||||
pub block_height: Slot,
|
||||
pub slot_deltas: Vec<BankSlotDelta>,
|
||||
pub snapshot_links: TempDir,
|
||||
pub storages: SnapshotStorages,
|
||||
pub hash: Hash, // temporarily here while we still have to calculate hash before serializing bank
|
||||
pub archive_format: ArchiveFormat,
|
||||
pub snapshot_version: SnapshotVersion,
|
||||
pub snapshot_output_dir: PathBuf,
|
||||
pub expected_capitalization: u64,
|
||||
pub hash_for_testing: Option<Hash>,
|
||||
pub simple_capitalization_testing: bool,
|
||||
}
|
||||
|
||||
impl AccountsPackagePre {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
slot: Slot,
|
||||
block_height: u64,
|
||||
slot_deltas: Vec<BankSlotDelta>,
|
||||
snapshot_links: TempDir,
|
||||
storages: SnapshotStorages,
|
||||
hash: Hash,
|
||||
archive_format: ArchiveFormat,
|
||||
snapshot_version: SnapshotVersion,
|
||||
snapshot_output_dir: PathBuf,
|
||||
expected_capitalization: u64,
|
||||
hash_for_testing: Option<Hash>,
|
||||
simple_capitalization_testing: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
slot,
|
||||
block_height,
|
||||
slot_deltas,
|
||||
snapshot_links,
|
||||
storages,
|
||||
hash,
|
||||
archive_format,
|
||||
snapshot_version,
|
||||
snapshot_output_dir,
|
||||
expected_capitalization,
|
||||
hash_for_testing,
|
||||
simple_capitalization_testing,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AccountsPackage {
|
||||
pub slot: Slot,
|
||||
pub block_height: Slot,
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
use crate::{
|
||||
accounts_db::AccountsDB,
|
||||
accounts_index::AccountIndex,
|
||||
bank::{Bank, BankSlotDelta, Builtins},
|
||||
bank_forks::ArchiveFormat,
|
||||
|
@ -6,7 +7,9 @@ use crate::{
|
|||
serde_snapshot::{
|
||||
bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
|
||||
},
|
||||
snapshot_package::{AccountsPackage, AccountsPackageSendError, AccountsPackageSender},
|
||||
snapshot_package::{
|
||||
AccountsPackage, AccountsPackagePre, AccountsPackageSendError, AccountsPackageSender,
|
||||
},
|
||||
};
|
||||
use bincode::{config::Options, serialize_into};
|
||||
use bzip2::bufread::BzDecoder;
|
||||
|
@ -144,7 +147,8 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||
snapshot_storages: SnapshotStorages,
|
||||
archive_format: ArchiveFormat,
|
||||
snapshot_version: SnapshotVersion,
|
||||
) -> Result<AccountsPackage> {
|
||||
hash_for_testing: Option<Hash>,
|
||||
) -> Result<AccountsPackagePre> {
|
||||
// Hard link all the snapshots we need for this package
|
||||
let snapshot_tmpdir = tempfile::Builder::new()
|
||||
.prefix(&format!("{}{}-", TMP_SNAPSHOT_PREFIX, bank.slot()))
|
||||
|
@ -169,22 +173,19 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||
)?;
|
||||
}
|
||||
|
||||
let snapshot_package_output_file = get_snapshot_archive_path(
|
||||
&snapshot_package_output_path,
|
||||
&(bank.slot(), bank.get_accounts_hash()),
|
||||
archive_format,
|
||||
);
|
||||
|
||||
let package = AccountsPackage::new(
|
||||
let package = AccountsPackagePre::new(
|
||||
bank.slot(),
|
||||
bank.block_height(),
|
||||
status_cache_slot_deltas,
|
||||
snapshot_tmpdir,
|
||||
snapshot_storages,
|
||||
snapshot_package_output_file,
|
||||
bank.get_accounts_hash(),
|
||||
archive_format,
|
||||
snapshot_version,
|
||||
snapshot_package_output_path.as_ref().to_path_buf(),
|
||||
bank.capitalization(),
|
||||
hash_for_testing,
|
||||
bank.simple_capitalization_enabled(),
|
||||
);
|
||||
|
||||
Ok(package)
|
||||
|
@ -630,12 +631,12 @@ pub fn bank_from_archive<P: AsRef<Path>>(
|
|||
Ok(bank)
|
||||
}
|
||||
|
||||
pub fn get_snapshot_archive_path<P: AsRef<Path>>(
|
||||
snapshot_output_dir: P,
|
||||
pub fn get_snapshot_archive_path(
|
||||
snapshot_output_dir: PathBuf,
|
||||
snapshot_hash: &(Slot, Hash),
|
||||
archive_format: ArchiveFormat,
|
||||
) -> PathBuf {
|
||||
snapshot_output_dir.as_ref().join(format!(
|
||||
snapshot_output_dir.join(format!(
|
||||
"snapshot-{}-{}{}",
|
||||
snapshot_hash.0,
|
||||
snapshot_hash.1,
|
||||
|
@ -886,6 +887,7 @@ pub fn snapshot_bank(
|
|||
snapshot_package_output_path: &Path,
|
||||
snapshot_version: SnapshotVersion,
|
||||
archive_format: &ArchiveFormat,
|
||||
hash_for_testing: Option<Hash>,
|
||||
) -> Result<()> {
|
||||
let storages: Vec<_> = root_bank.get_snapshot_storages();
|
||||
let mut add_snapshot_time = Measure::start("add-snapshot-ms");
|
||||
|
@ -908,6 +910,7 @@ pub fn snapshot_bank(
|
|||
storages,
|
||||
*archive_format,
|
||||
snapshot_version,
|
||||
hash_for_testing,
|
||||
)?;
|
||||
|
||||
accounts_package_sender.send(package)?;
|
||||
|
@ -946,12 +949,55 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||
storages,
|
||||
archive_format,
|
||||
snapshot_version,
|
||||
None,
|
||||
)?;
|
||||
|
||||
let package = process_accounts_package_pre(package);
|
||||
|
||||
archive_snapshot_package(&package)?;
|
||||
Ok(package.tar_output_file)
|
||||
}
|
||||
|
||||
pub fn process_accounts_package_pre(accounts_package: AccountsPackagePre) -> AccountsPackage {
|
||||
let mut time = Measure::start("hash");
|
||||
|
||||
let hash = accounts_package.hash; // temporarily remaining here
|
||||
if let Some(expected_hash) = accounts_package.hash_for_testing {
|
||||
let (hash, lamports) = AccountsDB::calculate_accounts_hash_without_index(
|
||||
accounts_package.storages.clone(),
|
||||
accounts_package.simple_capitalization_testing,
|
||||
);
|
||||
time.stop();
|
||||
|
||||
assert_eq!(accounts_package.expected_capitalization, lamports);
|
||||
|
||||
assert_eq!(expected_hash, hash);
|
||||
};
|
||||
|
||||
datapoint_info!(
|
||||
"accounts_hash_verifier",
|
||||
("calculate_hash", time.as_us(), i64),
|
||||
);
|
||||
|
||||
let tar_output_file = get_snapshot_archive_path(
|
||||
accounts_package.snapshot_output_dir,
|
||||
&(accounts_package.slot, hash),
|
||||
accounts_package.archive_format,
|
||||
);
|
||||
|
||||
AccountsPackage::new(
|
||||
accounts_package.slot,
|
||||
accounts_package.block_height,
|
||||
accounts_package.slot_deltas,
|
||||
accounts_package.snapshot_links,
|
||||
accounts_package.storages,
|
||||
tar_output_file,
|
||||
hash,
|
||||
accounts_package.archive_format,
|
||||
accounts_package.snapshot_version,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
@ -1418,6 +1418,11 @@ pub fn main() {
|
|||
.long("no-accounts-db-caching")
|
||||
.help("Disables accounts caching"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("accounts_db_test_hash_calculation")
|
||||
.long("accounts-db-test-hash-calculation")
|
||||
.help("Enables testing of hash calculation using stores in AccountsHashVerifier. This has a computational cost."),
|
||||
)
|
||||
.arg(
|
||||
// legacy nop argument
|
||||
Arg::with_name("accounts_db_caching_enabled")
|
||||
|
@ -1623,6 +1628,7 @@ pub fn main() {
|
|||
.unwrap_or(poh_service::DEFAULT_PINNED_CPU_CORE),
|
||||
account_indexes,
|
||||
accounts_db_caching_enabled: !matches.is_present("no_accounts_db_caching"),
|
||||
accounts_db_test_hash_calculation: matches.is_present("accounts_db_test_hash_calculation"),
|
||||
..ValidatorConfig::default()
|
||||
};
|
||||
|
||||
|
|
Loading…
Reference in New Issue