blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

merkle root code no longer adds lamports (#15298)

This commit is contained in:
Jeff Washington (jwash) 2021-02-12 17:30:14 -06:00 committed by GitHub
parent 2e7aebf0bb
commit b8448f4189
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 57 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
merkle-root-bench/src/main.rs
View File

@ -29,24 +29,24 @@ fn main() {
let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000); let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
let iterations = value_t!(matches, "iterations", usize).unwrap_or(20); let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
let hashes: Vec<_> = (0..num_accounts) let hashes: Vec<_> = (0..num_accounts)
.map(|_| (Pubkey::new_unique(), Hash::new_unique(), 1)) .map(|_| (Pubkey::new_unique(), Hash::new_unique()))
.collect(); .collect();
let elapsed: Vec<_> = (0..iterations) let elapsed: Vec<_> = (0..iterations)
.map(|_| { .map(|_| {
let hashes = hashes.clone(); // done outside timing let hashes = hashes.clone(); // done outside timing
let mut time = Measure::start("compute_merkle_root_and_capitalization"); let mut time = Measure::start("compute_merkle_root");
let fanout = 16; let fanout = 16;
AccountsDB::compute_merkle_root_and_capitalization(hashes, fanout); AccountsDB::compute_merkle_root(hashes, fanout);
time.stop(); time.stop();
time.as_us() time.as_us()
}) })
.collect(); .collect();
for result in &elapsed { for result in &elapsed {
println!("compute_merkle_root_and_capitalization(us),{}", result); println!("compute_merkle_root(us),{}", result);
} }
println!( println!(
"compute_merkle_root_and_capitalization(us) avg: {}", "compute_merkle_root(us) avg: {}",
elapsed.into_iter().sum::<u64>() as f64 / iterations as f64 elapsed.into_iter().sum::<u64>() as f64 / iterations as f64
); );
} }

137
runtime/src/accounts_db.rs
View File

@ -3482,25 +3482,13 @@ impl AccountsDB {
); );
} }
pub fn compute_merkle_root_and_capitalization(
hashes: Vec<(Pubkey, Hash, u64)>,
fanout: usize,
) -> (Hash, u64) {
Self::compute_merkle_root_and_capitalization_loop(hashes, fanout, |t| (t.1, t.2))
}
pub fn compute_merkle_root(hashes: Vec<(Pubkey, Hash)>, fanout: usize) -> Hash { pub fn compute_merkle_root(hashes: Vec<(Pubkey, Hash)>, fanout: usize) -> Hash {
Self::compute_merkle_root_and_capitalization_loop(hashes, fanout, |t| (t.1, 0)).0 Self::compute_merkle_root_loop(hashes, fanout, |t| t.1)
} }
// this function avoids an infinite recursion compiler error // this function avoids an infinite recursion compiler error
fn compute_merkle_root_and_capitalization_recurse( fn compute_merkle_root_recurse(hashes: Vec<Hash>, fanout: usize) -> Hash {
hashes: Vec<(Hash, u64)>, Self::compute_merkle_root_loop(hashes, fanout, |t: &Hash| *t)
fanout: usize,
) -> (Hash, u64) {
Self::compute_merkle_root_and_capitalization_loop(hashes, fanout, |t: &(Hash, u64)| {
(t.0, t.1)
})
} }
fn div_ceil(x: usize, y: usize) -> usize { fn div_ceil(x: usize, y: usize) -> usize {
@ -3513,17 +3501,13 @@ impl AccountsDB {
// For the first iteration, there could be more items in the tuple than just hash and lamports. // For the first iteration, there could be more items in the tuple than just hash and lamports.
// Using extractor allows us to avoid an unnecessary array copy on the first iteration. // Using extractor allows us to avoid an unnecessary array copy on the first iteration.
fn compute_merkle_root_and_capitalization_loop<T, F>( fn compute_merkle_root_loop<T, F>(hashes: Vec<T>, fanout: usize, extractor: F) -> Hash
hashes: Vec<T>,
fanout: usize,
extractor: F,
) -> (Hash, u64)
where where
F: Fn(&T) -> (Hash, u64) + std::marker::Sync, F: Fn(&T) -> Hash + std::marker::Sync,
T: std::marker::Sync, T: std::marker::Sync,
{ {
if hashes.is_empty() { if hashes.is_empty() {
return (Hasher::default().result(), 0); return Hasher::default().result();
} }
let mut time = Measure::start("time"); let mut time = Measure::start("time");
@ -3538,17 +3522,12 @@ impl AccountsDB {
let end_index = std::cmp::min(start_index + fanout, total_hashes); let end_index = std::cmp::min(start_index + fanout, total_hashes);
let mut hasher = Hasher::default(); let mut hasher = Hasher::default();
let mut this_sum = 0u128;
for item in hashes.iter().take(end_index).skip(start_index) { for item in hashes.iter().take(end_index).skip(start_index) {
let (h, l) = extractor(&item); let h = extractor(&item);
this_sum += l as u128;
hasher.hash(h.as_ref()); hasher.hash(h.as_ref());
} }
( hasher.result()
hasher.result(),
Self::checked_cast_for_capitalization(this_sum),
)
}) })
.collect(); .collect();
time.stop(); time.stop();
@ -3557,7 +3536,7 @@ impl AccountsDB {
if result.len() == 1 { if result.len() == 1 {
result[0] result[0]
} else { } else {
Self::compute_merkle_root_and_capitalization_recurse(result, fanout) Self::compute_merkle_root_recurse(result, fanout)
} }
} }
@ -3606,26 +3585,23 @@ impl AccountsDB {
data_index += 1; data_index += 1;
} }
(hasher.result(), 0) hasher.result()
}) })
.collect(); .collect();
time.stop(); time.stop();
debug!("hashing {} {}", total_hashes, time); debug!("hashing {} {}", total_hashes, time);
if result.len() == 1 { if result.len() == 1 {
result[0].0 result[0]
} else { } else {
Self::compute_merkle_root_and_capitalization_recurse(result, fanout).0 Self::compute_merkle_root_recurse(result, fanout)
} }
} }
fn accumulate_account_hashes(mut hashes: Vec<(Pubkey, Hash)>) -> Hash { fn accumulate_account_hashes(mut hashes: Vec<(Pubkey, Hash)>) -> Hash {
Self::sort_hashes_by_pubkey(&mut hashes); Self::sort_hashes_by_pubkey(&mut hashes);
let res = Self::compute_merkle_root_loop(hashes, MERKLE_FANOUT, |i| i.1)
Self::compute_merkle_root_and_capitalization_loop(hashes, MERKLE_FANOUT, |i| (i.1, 0));
res.0
} }
fn sort_hashes_by_pubkey(hashes: &mut Vec<(Pubkey, Hash)>) { fn sort_hashes_by_pubkey(hashes: &mut Vec<(Pubkey, Hash)>) {
@ -5664,12 +5640,7 @@ pub mod tests {
.into_iter() .into_iter()
.map(|i| Hash::new(&[(i) as u8; 32])) .map(|i| Hash::new(&[(i) as u8; 32]))
.collect(); .collect();
let expected = AccountsDB::compute_merkle_root_and_capitalization_loop( let expected = AccountsDB::compute_merkle_root_loop(hashes.clone(), MERKLE_FANOUT, |i| *i);
hashes.clone(),
MERKLE_FANOUT,
|i| (*i, 0),
)
.0;
assert_eq!( assert_eq!(
AccountsDB::flatten_hashes_and_hash( AccountsDB::flatten_hashes_and_hash(
@ -5880,49 +5851,45 @@ pub mod tests {
assert_eq!(stats.unreduced_entries, expected.len()); assert_eq!(stats.unreduced_entries, expected.len());
} }
fn sort_hashes_and_lamports_by_pubkey(hashes: &mut Vec<(Pubkey, Hash, u64)>) { fn test_hashing_larger(hashes: Vec<(Pubkey, Hash)>, fanout: usize) -> Hash {
hashes.par_sort_unstable_by(|a, b| a.0.cmp(&b.0)); let result = AccountsDB::compute_merkle_root(hashes.clone(), fanout);
}
fn test_hashing_larger(hashes: Vec<(Pubkey, Hash, u64)>, fanout: usize) -> (Hash, u64) {
let result = AccountsDB::compute_merkle_root_and_capitalization(hashes.clone(), fanout);
if hashes.len() >= fanout * fanout * fanout { if hashes.len() >= fanout * fanout * fanout {
let reduced: Vec<_> = hashes.iter().map(|x| x.1).collect(); let reduced: Vec<_> = hashes.iter().map(|x| x.1).collect();
let result2 = let result2 =
AccountsDB::compute_merkle_root_from_slices(hashes.len(), fanout, |start| { AccountsDB::compute_merkle_root_from_slices(hashes.len(), fanout, |start| {
&reduced[start..] &reduced[start..]
}); });
assert_eq!(result.0, result2); assert_eq!(result, result2);
let reduced2: Vec<_> = hashes.iter().map(|x| vec![x.1]).collect(); let reduced2: Vec<_> = hashes.iter().map(|x| vec![x.1]).collect();
let result2 = let result2 =
AccountsDB::flatten_hashes_and_hash(reduced2, fanout, &mut HashStats::default()); AccountsDB::flatten_hashes_and_hash(reduced2, fanout, &mut HashStats::default());
assert_eq!(result.0, result2); assert_eq!(result, result2);
} }
result result
} }
fn test_hashing(hashes: Vec<Hash>, fanout: usize) -> (Hash, u64) { fn test_hashing(hashes: Vec<Hash>, fanout: usize) -> Hash {
let temp: Vec<_> = hashes.iter().map(|h| (Pubkey::default(), *h, 0)).collect(); let temp: Vec<_> = hashes.iter().map(|h| (Pubkey::default(), *h)).collect();
let result = AccountsDB::compute_merkle_root_and_capitalization(temp, fanout); let result = AccountsDB::compute_merkle_root(temp, fanout);
if hashes.len() >= fanout * fanout * fanout { if hashes.len() >= fanout * fanout * fanout {
let reduced: Vec<_> = hashes.clone(); let reduced: Vec<_> = hashes.clone();
let result2 = let result2 =
AccountsDB::compute_merkle_root_from_slices(hashes.len(), fanout, |start| { AccountsDB::compute_merkle_root_from_slices(hashes.len(), fanout, |start| {
&reduced[start..] &reduced[start..]
}); });
assert_eq!(result.0, result2, "len: {}", hashes.len()); assert_eq!(result, result2, "len: {}", hashes.len());
let reduced2: Vec<_> = hashes.iter().map(|x| vec![*x]).collect(); let reduced2: Vec<_> = hashes.iter().map(|x| vec![*x]).collect();
let result2 = let result2 =
AccountsDB::flatten_hashes_and_hash(reduced2, fanout, &mut HashStats::default()); AccountsDB::flatten_hashes_and_hash(reduced2, fanout, &mut HashStats::default());
assert_eq!(result.0, result2, "len: {}", hashes.len()); assert_eq!(result, result2, "len: {}", hashes.len());
} }
result result
} }
#[test] #[test]
fn test_accountsdb_compute_merkle_root_and_capitalization_large() { fn test_accountsdb_compute_merkle_root_large() {
solana_logger::setup(); solana_logger::setup();
let mut num = 100; let mut num = 100;
@ -5935,7 +5902,7 @@ pub mod tests {
} }
#[test] #[test]
fn test_accountsdb_compute_merkle_root_and_capitalization() { fn test_accountsdb_compute_merkle_root() {
solana_logger::setup(); solana_logger::setup();
let expected_results = vec![ let expected_results = vec![
@ -5977,7 +5944,7 @@ pub mod tests {
.map(|i| { .map(|i| {
let key = Pubkey::new(&[(pass * iterations + count) as u8; 32]); let key = Pubkey::new(&[(pass * iterations + count) as u8; 32]);
let hash = Hash::new(&[(pass * iterations + count + i + 1) as u8; 32]); let hash = Hash::new(&[(pass * iterations + count + i + 1) as u8; 32]);
(key, hash, i as u64) (key, hash)
}) })
.collect(); .collect();
@ -5988,31 +5955,19 @@ pub mod tests {
let early_result = AccountsDB::accumulate_account_hashes( let early_result = AccountsDB::accumulate_account_hashes(
input.iter().map(|i| (i.0, i.1)).collect::<Vec<_>>(), input.iter().map(|i| (i.0, i.1)).collect::<Vec<_>>(),
); );
sort_hashes_and_lamports_by_pubkey(&mut input); AccountsDB::sort_hashes_by_pubkey(&mut input);
let result = let result = AccountsDB::compute_merkle_root(input.clone(), fanout);
AccountsDB::compute_merkle_root_and_capitalization(input.clone(), fanout); assert_eq!(early_result, result);
assert_eq!(early_result, result.0);
result result
}; };
let mut expected = 0;
if count > 0 {
let count = count as u64;
let last_number = count - 1;
expected = count * last_number / 2;
}
// compare against calculated result for lamports
assert_eq!(
result.1,
expected,
"failed at size: {}, with inputs: {:?}",
count,
input.into_iter().map(|x| x.2).collect::<Vec<u64>>()
);
// compare against captured, expected results for hash (and lamports) // compare against captured, expected results for hash (and lamports)
assert_eq!( assert_eq!(
(pass, count, &*(result.0.to_string()), result.1), (
pass,
count,
&*(result.to_string()),
expected_results[expected_index].3
), // we no longer calculate lamports
expected_results[expected_index] expected_results[expected_index]
); );
expected_index += 1; expected_index += 1;
@ -6022,15 +5977,27 @@ pub mod tests {
#[test] #[test]
#[should_panic(expected = "overflow is detected while summing capitalization")] #[should_panic(expected = "overflow is detected while summing capitalization")]
fn test_accountsdb_compute_merkle_root_and_capitalization_overflow() { fn test_accountsdb_lamport_overflow() {
solana_logger::setup(); solana_logger::setup();
let fanout = 2; let offset = 2;
let input = vec![ let input = vec![
(Pubkey::new_unique(), Hash::new_unique(), u64::MAX), CalculateHashIntermediate::new(
(Pubkey::new_unique(), Hash::new_unique(), 1), 0,
Hash::new_unique(),
u64::MAX - offset,
0,
Pubkey::new_unique(),
),
CalculateHashIntermediate::new(
0,
Hash::new_unique(),
offset + 1,
0,
Pubkey::new_unique(),
),
]; ];
AccountsDB::compute_merkle_root_and_capitalization(input, fanout); AccountsDB::de_dup_and_eliminate_zeros(input, &mut HashStats::default());
} }
#[test] #[test]