remove legacy merkle root (#14772)
* remove legacy merkle root f78197a * clippy * compile error * borrow error * derministic results * clippy * borrow
This commit is contained in:
parent
d1df9da7d3
commit
1b85114a9c
|
@ -28,38 +28,25 @@ fn main() {
|
||||||
|
|
||||||
let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
|
let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
|
||||||
let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
|
let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
|
||||||
let mut elapsed: Vec<u64> = vec![0; iterations];
|
|
||||||
let mut elapsed_legacy: Vec<u64> = vec![0; iterations];
|
|
||||||
let hashes: Vec<_> = (0..num_accounts)
|
let hashes: Vec<_> = (0..num_accounts)
|
||||||
.map(|_| (Pubkey::new_unique(), Hash::new_unique(), 1))
|
.map(|_| (Pubkey::new_unique(), Hash::new_unique(), 1))
|
||||||
.collect();
|
.collect();
|
||||||
for x in 0..iterations {
|
let elapsed: Vec<_> = (0..iterations)
|
||||||
let hashes = (hashes.clone(), hashes.clone()); // done outside timing
|
.map(|_| {
|
||||||
let mut time = Measure::start("compute_merkle_root_and_capitalization");
|
let hashes = hashes.clone(); // done outside timing
|
||||||
let fanout = 16;
|
let mut time = Measure::start("compute_merkle_root_and_capitalization");
|
||||||
let results = AccountsDB::compute_merkle_root_and_capitalization(hashes.0, fanout);
|
let fanout = 16;
|
||||||
time.stop();
|
AccountsDB::compute_merkle_root_and_capitalization(hashes, fanout);
|
||||||
let mut time_legacy = Measure::start("hash");
|
time.stop();
|
||||||
let results_hash = AccountsDB::compute_merkle_root_legacy(hashes.1, fanout);
|
time.as_us()
|
||||||
time_legacy.stop();
|
})
|
||||||
assert_eq!(results_hash, results.0);
|
.collect();
|
||||||
elapsed[x] = time.as_us();
|
|
||||||
elapsed_legacy[x] = time_legacy.as_us();
|
|
||||||
}
|
|
||||||
|
|
||||||
let len = elapsed.len();
|
for result in &elapsed {
|
||||||
for x in 0..iterations {
|
println!("compute_merkle_root_and_capitalization(us),{}", result);
|
||||||
println!(
|
|
||||||
"compute_merkle_root_and_capitalization(us),{},legacy(us),{}",
|
|
||||||
elapsed[x], elapsed_legacy[x]
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
println!(
|
println!(
|
||||||
"compute_merkle_root_and_capitalization(us) avg: {}",
|
"compute_merkle_root_and_capitalization(us) avg: {}",
|
||||||
elapsed.into_iter().sum::<u64>() as f64 / len as f64
|
elapsed.into_iter().sum::<u64>() as f64 / iterations as f64
|
||||||
);
|
|
||||||
println!(
|
|
||||||
"compute_merkle_root_legacy(us) avg: {}",
|
|
||||||
elapsed_legacy.into_iter().sum::<u64>() as f64 / len as f64
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
@ -3371,36 +3371,6 @@ impl AccountsDB {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: remove pub. bench uses it for now.
|
|
||||||
pub fn compute_merkle_root_legacy(hashes: Vec<(Pubkey, Hash, u64)>, fanout: usize) -> Hash {
|
|
||||||
let hashes: Vec<_> = hashes
|
|
||||||
.into_iter()
|
|
||||||
.map(|(_pubkey, hash, _lamports)| hash)
|
|
||||||
.collect();
|
|
||||||
let mut hashes: Vec<_> = hashes.chunks(fanout).map(|x| x.to_vec()).collect();
|
|
||||||
while hashes.len() > 1 {
|
|
||||||
let mut time = Measure::start("time");
|
|
||||||
let new_hashes: Vec<Hash> = hashes
|
|
||||||
.par_iter()
|
|
||||||
.map(|h| {
|
|
||||||
let mut hasher = Hasher::default();
|
|
||||||
for v in h.iter() {
|
|
||||||
hasher.hash(v.as_ref());
|
|
||||||
}
|
|
||||||
hasher.result()
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
time.stop();
|
|
||||||
debug!("hashing {} {}", hashes.len(), time);
|
|
||||||
hashes = new_hashes.chunks(fanout).map(|x| x.to_vec()).collect();
|
|
||||||
}
|
|
||||||
let mut hasher = Hasher::default();
|
|
||||||
hashes.into_iter().flatten().for_each(|hash| {
|
|
||||||
hasher.hash(hash.as_ref());
|
|
||||||
});
|
|
||||||
hasher.result()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn compute_merkle_root_and_capitalization(
|
pub fn compute_merkle_root_and_capitalization(
|
||||||
hashes: Vec<(Pubkey, Hash, u64)>,
|
hashes: Vec<(Pubkey, Hash, u64)>,
|
||||||
fanout: usize,
|
fanout: usize,
|
||||||
|
@ -3477,7 +3447,7 @@ impl AccountsDB {
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
debug: bool,
|
debug: bool,
|
||||||
) -> Hash {
|
) -> Hash {
|
||||||
let (hash, ..) = Self::accumulate_account_hashes_and_capitalization(hashes, slot, debug);
|
let (hash, ..) = Self::accumulate_account_hashes_and_capitalization(hashes, slot, debug).0;
|
||||||
hash
|
hash
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3489,7 +3459,7 @@ impl AccountsDB {
|
||||||
mut hashes: Vec<(Pubkey, Hash, u64)>,
|
mut hashes: Vec<(Pubkey, Hash, u64)>,
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
debug: bool,
|
debug: bool,
|
||||||
) -> (Hash, u64) {
|
) -> ((Hash, u64), (Measure, Measure)) {
|
||||||
let mut sort_time = Measure::start("sort");
|
let mut sort_time = Measure::start("sort");
|
||||||
Self::sort_hashes_by_pubkey(&mut hashes);
|
Self::sort_hashes_by_pubkey(&mut hashes);
|
||||||
sort_time.stop();
|
sort_time.stop();
|
||||||
|
@ -3505,12 +3475,7 @@ impl AccountsDB {
|
||||||
let res = Self::compute_merkle_root_and_capitalization(hashes, fanout);
|
let res = Self::compute_merkle_root_and_capitalization(hashes, fanout);
|
||||||
hash_time.stop();
|
hash_time.stop();
|
||||||
|
|
||||||
debug!(
|
(res, (sort_time, hash_time))
|
||||||
"accumulate_account_hashes_and_capitalization: {},{}",
|
|
||||||
sort_time, hash_time
|
|
||||||
);
|
|
||||||
|
|
||||||
res
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn checked_cast_for_capitalization(balance: u128) -> u64 {
|
pub fn checked_cast_for_capitalization(balance: u128) -> u64 {
|
||||||
|
@ -3627,13 +3592,15 @@ impl AccountsDB {
|
||||||
let hash_total = hashes.len();
|
let hash_total = hashes.len();
|
||||||
|
|
||||||
let mut accumulate = Measure::start("accumulate");
|
let mut accumulate = Measure::start("accumulate");
|
||||||
let (accumulated_hash, total_lamports) =
|
let ((accumulated_hash, total_lamports), (sort_time, hash_time)) =
|
||||||
Self::accumulate_account_hashes_and_capitalization(hashes, slot, false);
|
Self::accumulate_account_hashes_and_capitalization(hashes, slot, false);
|
||||||
accumulate.stop();
|
accumulate.stop();
|
||||||
datapoint_info!(
|
datapoint_info!(
|
||||||
"update_accounts_hash",
|
"update_accounts_hash",
|
||||||
("accounts_scan", scan.as_us(), i64),
|
("accounts_scan", scan.as_us(), i64),
|
||||||
("hash_accumulate", accumulate.as_us(), i64),
|
("hash_accumulate", accumulate.as_us(), i64),
|
||||||
|
("hash", hash_time.as_us(), i64),
|
||||||
|
("sort", sort_time.as_us(), i64),
|
||||||
("hash_total", hash_total, i64),
|
("hash_total", hash_total, i64),
|
||||||
);
|
);
|
||||||
Ok((accumulated_hash, total_lamports))
|
Ok((accumulated_hash, total_lamports))
|
||||||
|
@ -4466,6 +4433,30 @@ pub mod tests {
|
||||||
fn test_accountsdb_compute_merkle_root_and_capitalization() {
|
fn test_accountsdb_compute_merkle_root_and_capitalization() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
|
|
||||||
|
let expected_results = vec![
|
||||||
|
(0, 0, "GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn", 0),
|
||||||
|
(0, 1, "8unXKJYTxrR423HgQxbDmx29mFri1QNrzVKKDxEfc6bj", 0),
|
||||||
|
(0, 2, "6QfkevXLLqbfAaR1kVjvMLFtEXvNUVrpmkwXqgsYtCFW", 1),
|
||||||
|
(0, 3, "G3FrJd9JrXcMiqChTSfvEdBL2sCPny3ebiUy9Xxbn7a2", 3),
|
||||||
|
(0, 4, "G3sZXHhwoCFuNyWy7Efffr47RBW33ibEp7b2hqNDmXdu", 6),
|
||||||
|
(0, 5, "78atJJYpokAPKMJwHxUW8SBDvPkkSpTBV7GiB27HwosJ", 10),
|
||||||
|
(0, 6, "7c9SM2BmCRVVXdrEdKcMK91MviPqXqQMd8QAb77tgLEy", 15),
|
||||||
|
(0, 7, "3hsmnZPhf22UvBLiZ4dVa21Qsdh65CCrtYXsb8MxoVAa", 21),
|
||||||
|
(0, 8, "5bwXUiC6RCRhb8fqvjvUXT6waU25str3UXA3a6Aq1jux", 28),
|
||||||
|
(0, 9, "3NNtQKH6PaYpCnFBtyi2icK9eYX3YM5pqA3SKaXtUNzu", 36),
|
||||||
|
(1, 0, "GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn", 0),
|
||||||
|
(1, 1, "4GWVCsnEu1iRyxjAB3F7J7C4MMvcoxFWtP9ihvwvDgxY", 0),
|
||||||
|
(1, 2, "8ML8Te6Uw2mipFr2v9sMZDcziXzhVqJo2qeMJohg1CJx", 1),
|
||||||
|
(1, 3, "AMEuC3AgqAeRBGBhSfTmuMdfbAiXJnGmKv99kHmcAE1H", 3),
|
||||||
|
(1, 4, "HEnDuJLHpsQfrApimGrovTqPEF6Vkrx2dKFr3BDtYzWx", 6),
|
||||||
|
(1, 5, "6rH69iP2yM1o565noZN1EqjySW4PhYUskz3c5tXePUfV", 10),
|
||||||
|
(1, 6, "7qEQMEXdfSPjbZ3q4cuuZwebDMvTvuaQ3dBiHoDUKo9a", 15),
|
||||||
|
(1, 7, "GDJz7LSKYjqqz6ujCaaQRJRmQ7TLNCwYJhdT84qT4qwk", 21),
|
||||||
|
(1, 8, "HT9krPLVTo3rr5WZQBQFrbqWs8SbYScXfnt8EVuobboM", 28),
|
||||||
|
(1, 9, "8y2pMgqMdRsvqw6BQXm6wtz3qxGPss72i6H6gVpPyeda", 36),
|
||||||
|
];
|
||||||
|
|
||||||
|
let mut expected_index = 0;
|
||||||
let start = 0;
|
let start = 0;
|
||||||
let default_fanout = 2;
|
let default_fanout = 2;
|
||||||
let fanout_in_accumulate = 16;
|
let fanout_in_accumulate = 16;
|
||||||
|
@ -4479,7 +4470,11 @@ pub mod tests {
|
||||||
};
|
};
|
||||||
for count in start..iterations {
|
for count in start..iterations {
|
||||||
let mut input: Vec<_> = (0..count)
|
let mut input: Vec<_> = (0..count)
|
||||||
.map(|i| (Pubkey::new_unique(), Hash::new_unique(), i as u64))
|
.map(|i| {
|
||||||
|
let key = Pubkey::new(&[(pass * iterations + count) as u8; 32]);
|
||||||
|
let hash = Hash::new(&[(pass * iterations + count + i + 1) as u8; 32]);
|
||||||
|
(key, hash, i as u64)
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
let result;
|
let result;
|
||||||
if pass == 0 {
|
if pass == 0 {
|
||||||
|
@ -4490,7 +4485,8 @@ pub mod tests {
|
||||||
input.clone(),
|
input.clone(),
|
||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
);
|
)
|
||||||
|
.0;
|
||||||
AccountsDB::sort_hashes_by_pubkey(&mut input);
|
AccountsDB::sort_hashes_by_pubkey(&mut input);
|
||||||
}
|
}
|
||||||
let mut expected = 0;
|
let mut expected = 0;
|
||||||
|
@ -4499,6 +4495,8 @@ pub mod tests {
|
||||||
let last_number = count - 1;
|
let last_number = count - 1;
|
||||||
expected = count * last_number / 2;
|
expected = count * last_number / 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// compare against calculated result for lamports
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
result.1,
|
result.1,
|
||||||
expected,
|
expected,
|
||||||
|
@ -4506,8 +4504,13 @@ pub mod tests {
|
||||||
count,
|
count,
|
||||||
input.into_iter().map(|x| x.2).collect::<Vec<u64>>()
|
input.into_iter().map(|x| x.2).collect::<Vec<u64>>()
|
||||||
);
|
);
|
||||||
let hash_result_legacy = AccountsDB::compute_merkle_root_legacy(input, fanout);
|
|
||||||
assert_eq!(result.0, hash_result_legacy, "failed at size: {}", count);
|
// compare against captured, expected results for hash (and lamports)
|
||||||
|
assert_eq!(
|
||||||
|
(pass, count, &*(result.0.to_string()), result.1),
|
||||||
|
expected_results[expected_index]
|
||||||
|
);
|
||||||
|
expected_index += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue