merkle root code no longer adds lamports (#15298)
This commit is contained in:
parent
2e7aebf0bb
commit
b8448f4189
|
@ -29,24 +29,24 @@ fn main() {
|
||||||
let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
|
let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
|
||||||
let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
|
let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
|
||||||
let hashes: Vec<_> = (0..num_accounts)
|
let hashes: Vec<_> = (0..num_accounts)
|
||||||
.map(|_| (Pubkey::new_unique(), Hash::new_unique(), 1))
|
.map(|_| (Pubkey::new_unique(), Hash::new_unique()))
|
||||||
.collect();
|
.collect();
|
||||||
let elapsed: Vec<_> = (0..iterations)
|
let elapsed: Vec<_> = (0..iterations)
|
||||||
.map(|_| {
|
.map(|_| {
|
||||||
let hashes = hashes.clone(); // done outside timing
|
let hashes = hashes.clone(); // done outside timing
|
||||||
let mut time = Measure::start("compute_merkle_root_and_capitalization");
|
let mut time = Measure::start("compute_merkle_root");
|
||||||
let fanout = 16;
|
let fanout = 16;
|
||||||
AccountsDB::compute_merkle_root_and_capitalization(hashes, fanout);
|
AccountsDB::compute_merkle_root(hashes, fanout);
|
||||||
time.stop();
|
time.stop();
|
||||||
time.as_us()
|
time.as_us()
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
for result in &elapsed {
|
for result in &elapsed {
|
||||||
println!("compute_merkle_root_and_capitalization(us),{}", result);
|
println!("compute_merkle_root(us),{}", result);
|
||||||
}
|
}
|
||||||
println!(
|
println!(
|
||||||
"compute_merkle_root_and_capitalization(us) avg: {}",
|
"compute_merkle_root(us) avg: {}",
|
||||||
elapsed.into_iter().sum::<u64>() as f64 / iterations as f64
|
elapsed.into_iter().sum::<u64>() as f64 / iterations as f64
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
@ -3482,25 +3482,13 @@ impl AccountsDB {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn compute_merkle_root_and_capitalization(
|
|
||||||
hashes: Vec<(Pubkey, Hash, u64)>,
|
|
||||||
fanout: usize,
|
|
||||||
) -> (Hash, u64) {
|
|
||||||
Self::compute_merkle_root_and_capitalization_loop(hashes, fanout, |t| (t.1, t.2))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn compute_merkle_root(hashes: Vec<(Pubkey, Hash)>, fanout: usize) -> Hash {
|
pub fn compute_merkle_root(hashes: Vec<(Pubkey, Hash)>, fanout: usize) -> Hash {
|
||||||
Self::compute_merkle_root_and_capitalization_loop(hashes, fanout, |t| (t.1, 0)).0
|
Self::compute_merkle_root_loop(hashes, fanout, |t| t.1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// this function avoids an infinite recursion compiler error
|
// this function avoids an infinite recursion compiler error
|
||||||
fn compute_merkle_root_and_capitalization_recurse(
|
fn compute_merkle_root_recurse(hashes: Vec<Hash>, fanout: usize) -> Hash {
|
||||||
hashes: Vec<(Hash, u64)>,
|
Self::compute_merkle_root_loop(hashes, fanout, |t: &Hash| *t)
|
||||||
fanout: usize,
|
|
||||||
) -> (Hash, u64) {
|
|
||||||
Self::compute_merkle_root_and_capitalization_loop(hashes, fanout, |t: &(Hash, u64)| {
|
|
||||||
(t.0, t.1)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn div_ceil(x: usize, y: usize) -> usize {
|
fn div_ceil(x: usize, y: usize) -> usize {
|
||||||
|
@ -3513,17 +3501,13 @@ impl AccountsDB {
|
||||||
|
|
||||||
// For the first iteration, there could be more items in the tuple than just hash and lamports.
|
// For the first iteration, there could be more items in the tuple than just hash and lamports.
|
||||||
// Using extractor allows us to avoid an unnecessary array copy on the first iteration.
|
// Using extractor allows us to avoid an unnecessary array copy on the first iteration.
|
||||||
fn compute_merkle_root_and_capitalization_loop<T, F>(
|
fn compute_merkle_root_loop<T, F>(hashes: Vec<T>, fanout: usize, extractor: F) -> Hash
|
||||||
hashes: Vec<T>,
|
|
||||||
fanout: usize,
|
|
||||||
extractor: F,
|
|
||||||
) -> (Hash, u64)
|
|
||||||
where
|
where
|
||||||
F: Fn(&T) -> (Hash, u64) + std::marker::Sync,
|
F: Fn(&T) -> Hash + std::marker::Sync,
|
||||||
T: std::marker::Sync,
|
T: std::marker::Sync,
|
||||||
{
|
{
|
||||||
if hashes.is_empty() {
|
if hashes.is_empty() {
|
||||||
return (Hasher::default().result(), 0);
|
return Hasher::default().result();
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut time = Measure::start("time");
|
let mut time = Measure::start("time");
|
||||||
|
@ -3538,17 +3522,12 @@ impl AccountsDB {
|
||||||
let end_index = std::cmp::min(start_index + fanout, total_hashes);
|
let end_index = std::cmp::min(start_index + fanout, total_hashes);
|
||||||
|
|
||||||
let mut hasher = Hasher::default();
|
let mut hasher = Hasher::default();
|
||||||
let mut this_sum = 0u128;
|
|
||||||
for item in hashes.iter().take(end_index).skip(start_index) {
|
for item in hashes.iter().take(end_index).skip(start_index) {
|
||||||
let (h, l) = extractor(&item);
|
let h = extractor(&item);
|
||||||
this_sum += l as u128;
|
|
||||||
hasher.hash(h.as_ref());
|
hasher.hash(h.as_ref());
|
||||||
}
|
}
|
||||||
|
|
||||||
(
|
hasher.result()
|
||||||
hasher.result(),
|
|
||||||
Self::checked_cast_for_capitalization(this_sum),
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
time.stop();
|
time.stop();
|
||||||
|
@ -3557,7 +3536,7 @@ impl AccountsDB {
|
||||||
if result.len() == 1 {
|
if result.len() == 1 {
|
||||||
result[0]
|
result[0]
|
||||||
} else {
|
} else {
|
||||||
Self::compute_merkle_root_and_capitalization_recurse(result, fanout)
|
Self::compute_merkle_root_recurse(result, fanout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3606,26 +3585,23 @@ impl AccountsDB {
|
||||||
data_index += 1;
|
data_index += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
(hasher.result(), 0)
|
hasher.result()
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
time.stop();
|
time.stop();
|
||||||
debug!("hashing {} {}", total_hashes, time);
|
debug!("hashing {} {}", total_hashes, time);
|
||||||
|
|
||||||
if result.len() == 1 {
|
if result.len() == 1 {
|
||||||
result[0].0
|
result[0]
|
||||||
} else {
|
} else {
|
||||||
Self::compute_merkle_root_and_capitalization_recurse(result, fanout).0
|
Self::compute_merkle_root_recurse(result, fanout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn accumulate_account_hashes(mut hashes: Vec<(Pubkey, Hash)>) -> Hash {
|
fn accumulate_account_hashes(mut hashes: Vec<(Pubkey, Hash)>) -> Hash {
|
||||||
Self::sort_hashes_by_pubkey(&mut hashes);
|
Self::sort_hashes_by_pubkey(&mut hashes);
|
||||||
|
|
||||||
let res =
|
Self::compute_merkle_root_loop(hashes, MERKLE_FANOUT, |i| i.1)
|
||||||
Self::compute_merkle_root_and_capitalization_loop(hashes, MERKLE_FANOUT, |i| (i.1, 0));
|
|
||||||
|
|
||||||
res.0
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sort_hashes_by_pubkey(hashes: &mut Vec<(Pubkey, Hash)>) {
|
fn sort_hashes_by_pubkey(hashes: &mut Vec<(Pubkey, Hash)>) {
|
||||||
|
@ -5664,12 +5640,7 @@ pub mod tests {
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|i| Hash::new(&[(i) as u8; 32]))
|
.map(|i| Hash::new(&[(i) as u8; 32]))
|
||||||
.collect();
|
.collect();
|
||||||
let expected = AccountsDB::compute_merkle_root_and_capitalization_loop(
|
let expected = AccountsDB::compute_merkle_root_loop(hashes.clone(), MERKLE_FANOUT, |i| *i);
|
||||||
hashes.clone(),
|
|
||||||
MERKLE_FANOUT,
|
|
||||||
|i| (*i, 0),
|
|
||||||
)
|
|
||||||
.0;
|
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
AccountsDB::flatten_hashes_and_hash(
|
AccountsDB::flatten_hashes_and_hash(
|
||||||
|
@ -5880,49 +5851,45 @@ pub mod tests {
|
||||||
assert_eq!(stats.unreduced_entries, expected.len());
|
assert_eq!(stats.unreduced_entries, expected.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sort_hashes_and_lamports_by_pubkey(hashes: &mut Vec<(Pubkey, Hash, u64)>) {
|
fn test_hashing_larger(hashes: Vec<(Pubkey, Hash)>, fanout: usize) -> Hash {
|
||||||
hashes.par_sort_unstable_by(|a, b| a.0.cmp(&b.0));
|
let result = AccountsDB::compute_merkle_root(hashes.clone(), fanout);
|
||||||
}
|
|
||||||
|
|
||||||
fn test_hashing_larger(hashes: Vec<(Pubkey, Hash, u64)>, fanout: usize) -> (Hash, u64) {
|
|
||||||
let result = AccountsDB::compute_merkle_root_and_capitalization(hashes.clone(), fanout);
|
|
||||||
if hashes.len() >= fanout * fanout * fanout {
|
if hashes.len() >= fanout * fanout * fanout {
|
||||||
let reduced: Vec<_> = hashes.iter().map(|x| x.1).collect();
|
let reduced: Vec<_> = hashes.iter().map(|x| x.1).collect();
|
||||||
let result2 =
|
let result2 =
|
||||||
AccountsDB::compute_merkle_root_from_slices(hashes.len(), fanout, |start| {
|
AccountsDB::compute_merkle_root_from_slices(hashes.len(), fanout, |start| {
|
||||||
&reduced[start..]
|
&reduced[start..]
|
||||||
});
|
});
|
||||||
assert_eq!(result.0, result2);
|
assert_eq!(result, result2);
|
||||||
|
|
||||||
let reduced2: Vec<_> = hashes.iter().map(|x| vec![x.1]).collect();
|
let reduced2: Vec<_> = hashes.iter().map(|x| vec![x.1]).collect();
|
||||||
let result2 =
|
let result2 =
|
||||||
AccountsDB::flatten_hashes_and_hash(reduced2, fanout, &mut HashStats::default());
|
AccountsDB::flatten_hashes_and_hash(reduced2, fanout, &mut HashStats::default());
|
||||||
assert_eq!(result.0, result2);
|
assert_eq!(result, result2);
|
||||||
}
|
}
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
fn test_hashing(hashes: Vec<Hash>, fanout: usize) -> (Hash, u64) {
|
fn test_hashing(hashes: Vec<Hash>, fanout: usize) -> Hash {
|
||||||
let temp: Vec<_> = hashes.iter().map(|h| (Pubkey::default(), *h, 0)).collect();
|
let temp: Vec<_> = hashes.iter().map(|h| (Pubkey::default(), *h)).collect();
|
||||||
let result = AccountsDB::compute_merkle_root_and_capitalization(temp, fanout);
|
let result = AccountsDB::compute_merkle_root(temp, fanout);
|
||||||
if hashes.len() >= fanout * fanout * fanout {
|
if hashes.len() >= fanout * fanout * fanout {
|
||||||
let reduced: Vec<_> = hashes.clone();
|
let reduced: Vec<_> = hashes.clone();
|
||||||
let result2 =
|
let result2 =
|
||||||
AccountsDB::compute_merkle_root_from_slices(hashes.len(), fanout, |start| {
|
AccountsDB::compute_merkle_root_from_slices(hashes.len(), fanout, |start| {
|
||||||
&reduced[start..]
|
&reduced[start..]
|
||||||
});
|
});
|
||||||
assert_eq!(result.0, result2, "len: {}", hashes.len());
|
assert_eq!(result, result2, "len: {}", hashes.len());
|
||||||
|
|
||||||
let reduced2: Vec<_> = hashes.iter().map(|x| vec![*x]).collect();
|
let reduced2: Vec<_> = hashes.iter().map(|x| vec![*x]).collect();
|
||||||
let result2 =
|
let result2 =
|
||||||
AccountsDB::flatten_hashes_and_hash(reduced2, fanout, &mut HashStats::default());
|
AccountsDB::flatten_hashes_and_hash(reduced2, fanout, &mut HashStats::default());
|
||||||
assert_eq!(result.0, result2, "len: {}", hashes.len());
|
assert_eq!(result, result2, "len: {}", hashes.len());
|
||||||
}
|
}
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_compute_merkle_root_and_capitalization_large() {
|
fn test_accountsdb_compute_merkle_root_large() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
|
|
||||||
let mut num = 100;
|
let mut num = 100;
|
||||||
|
@ -5935,7 +5902,7 @@ pub mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_accountsdb_compute_merkle_root_and_capitalization() {
|
fn test_accountsdb_compute_merkle_root() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
|
|
||||||
let expected_results = vec![
|
let expected_results = vec![
|
||||||
|
@ -5977,7 +5944,7 @@ pub mod tests {
|
||||||
.map(|i| {
|
.map(|i| {
|
||||||
let key = Pubkey::new(&[(pass * iterations + count) as u8; 32]);
|
let key = Pubkey::new(&[(pass * iterations + count) as u8; 32]);
|
||||||
let hash = Hash::new(&[(pass * iterations + count + i + 1) as u8; 32]);
|
let hash = Hash::new(&[(pass * iterations + count + i + 1) as u8; 32]);
|
||||||
(key, hash, i as u64)
|
(key, hash)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
@ -5988,31 +5955,19 @@ pub mod tests {
|
||||||
let early_result = AccountsDB::accumulate_account_hashes(
|
let early_result = AccountsDB::accumulate_account_hashes(
|
||||||
input.iter().map(|i| (i.0, i.1)).collect::<Vec<_>>(),
|
input.iter().map(|i| (i.0, i.1)).collect::<Vec<_>>(),
|
||||||
);
|
);
|
||||||
sort_hashes_and_lamports_by_pubkey(&mut input);
|
AccountsDB::sort_hashes_by_pubkey(&mut input);
|
||||||
let result =
|
let result = AccountsDB::compute_merkle_root(input.clone(), fanout);
|
||||||
AccountsDB::compute_merkle_root_and_capitalization(input.clone(), fanout);
|
assert_eq!(early_result, result);
|
||||||
assert_eq!(early_result, result.0);
|
|
||||||
result
|
result
|
||||||
};
|
};
|
||||||
let mut expected = 0;
|
|
||||||
if count > 0 {
|
|
||||||
let count = count as u64;
|
|
||||||
let last_number = count - 1;
|
|
||||||
expected = count * last_number / 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
// compare against calculated result for lamports
|
|
||||||
assert_eq!(
|
|
||||||
result.1,
|
|
||||||
expected,
|
|
||||||
"failed at size: {}, with inputs: {:?}",
|
|
||||||
count,
|
|
||||||
input.into_iter().map(|x| x.2).collect::<Vec<u64>>()
|
|
||||||
);
|
|
||||||
|
|
||||||
// compare against captured, expected results for hash (and lamports)
|
// compare against captured, expected results for hash (and lamports)
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
(pass, count, &*(result.0.to_string()), result.1),
|
(
|
||||||
|
pass,
|
||||||
|
count,
|
||||||
|
&*(result.to_string()),
|
||||||
|
expected_results[expected_index].3
|
||||||
|
), // we no longer calculate lamports
|
||||||
expected_results[expected_index]
|
expected_results[expected_index]
|
||||||
);
|
);
|
||||||
expected_index += 1;
|
expected_index += 1;
|
||||||
|
@ -6022,15 +5977,27 @@ pub mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[should_panic(expected = "overflow is detected while summing capitalization")]
|
#[should_panic(expected = "overflow is detected while summing capitalization")]
|
||||||
fn test_accountsdb_compute_merkle_root_and_capitalization_overflow() {
|
fn test_accountsdb_lamport_overflow() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
|
|
||||||
let fanout = 2;
|
let offset = 2;
|
||||||
let input = vec![
|
let input = vec![
|
||||||
(Pubkey::new_unique(), Hash::new_unique(), u64::MAX),
|
CalculateHashIntermediate::new(
|
||||||
(Pubkey::new_unique(), Hash::new_unique(), 1),
|
0,
|
||||||
|
Hash::new_unique(),
|
||||||
|
u64::MAX - offset,
|
||||||
|
0,
|
||||||
|
Pubkey::new_unique(),
|
||||||
|
),
|
||||||
|
CalculateHashIntermediate::new(
|
||||||
|
0,
|
||||||
|
Hash::new_unique(),
|
||||||
|
offset + 1,
|
||||||
|
0,
|
||||||
|
Pubkey::new_unique(),
|
||||||
|
),
|
||||||
];
|
];
|
||||||
AccountsDB::compute_merkle_root_and_capitalization(input, fanout);
|
AccountsDB::de_dup_and_eliminate_zeros(input, &mut HashStats::default());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
Loading…
Reference in New Issue