Cleanup entry.rs packing code (#3303)
This commit is contained in:
parent
be5f800390
commit
36763d0802
|
@ -3,6 +3,7 @@
|
||||||
//! can do its processing in parallel with signature verification on the GPU.
|
//! can do its processing in parallel with signature verification on the GPU.
|
||||||
|
|
||||||
use crate::cluster_info::ClusterInfo;
|
use crate::cluster_info::ClusterInfo;
|
||||||
|
use crate::entry;
|
||||||
use crate::entry::Entry;
|
use crate::entry::Entry;
|
||||||
use crate::leader_confirmation_service::LeaderConfirmationService;
|
use crate::leader_confirmation_service::LeaderConfirmationService;
|
||||||
use crate::leader_schedule_utils;
|
use crate::leader_schedule_utils;
|
||||||
|
@ -288,7 +289,12 @@ impl BankingStage {
|
||||||
) -> Result<(usize)> {
|
) -> Result<(usize)> {
|
||||||
let mut chunk_start = 0;
|
let mut chunk_start = 0;
|
||||||
while chunk_start != transactions.len() {
|
while chunk_start != transactions.len() {
|
||||||
let chunk_end = chunk_start + Entry::num_will_fit(&transactions[chunk_start..]);
|
let chunk_end = chunk_start
|
||||||
|
+ entry::num_will_fit(
|
||||||
|
&transactions[chunk_start..],
|
||||||
|
packet::BLOB_DATA_SIZE as u64,
|
||||||
|
&Entry::serialized_size,
|
||||||
|
);
|
||||||
|
|
||||||
let result = Self::process_and_record_transactions(
|
let result = Self::process_and_record_transactions(
|
||||||
bank,
|
bank,
|
||||||
|
|
|
@ -122,42 +122,6 @@ impl Entry {
|
||||||
(2 * size_of::<u64>() + size_of::<Hash>()) as u64 + txs_size
|
(2 * size_of::<u64>() + size_of::<Hash>()) as u64 + txs_size
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn num_will_fit(transactions: &[Transaction]) -> usize {
|
|
||||||
if transactions.is_empty() {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
let mut num = transactions.len();
|
|
||||||
let mut upper = transactions.len();
|
|
||||||
let mut lower = 1; // if one won't fit, we have a lot of TODOs
|
|
||||||
let mut next = transactions.len(); // optimistic
|
|
||||||
loop {
|
|
||||||
debug!(
|
|
||||||
"num {}, upper {} lower {} next {} transactions.len() {}",
|
|
||||||
num,
|
|
||||||
upper,
|
|
||||||
lower,
|
|
||||||
next,
|
|
||||||
transactions.len()
|
|
||||||
);
|
|
||||||
if Self::serialized_size(&transactions[..num]) <= BLOB_DATA_SIZE as u64 {
|
|
||||||
next = (upper + num) / 2;
|
|
||||||
lower = num;
|
|
||||||
debug!("num {} fits, maybe too well? trying {}", num, next);
|
|
||||||
} else {
|
|
||||||
next = (lower + num) / 2;
|
|
||||||
upper = num;
|
|
||||||
debug!("num {} doesn't fit! trying {}", num, next);
|
|
||||||
}
|
|
||||||
// same as last time
|
|
||||||
if next == num {
|
|
||||||
debug!("converged on num {}", num);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
num = next;
|
|
||||||
}
|
|
||||||
num
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Creates the next Tick Entry `num_hashes` after `start_hash`.
|
/// Creates the next Tick Entry `num_hashes` after `start_hash`.
|
||||||
pub fn new_mut(
|
pub fn new_mut(
|
||||||
start_hash: &mut Hash,
|
start_hash: &mut Hash,
|
||||||
|
@ -303,6 +267,67 @@ pub fn next_entry_mut(start: &mut Hash, num_hashes: u64, transactions: Vec<Trans
|
||||||
entry
|
entry
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn num_will_fit<T, F>(serializables: &[T], max_size: u64, serialized_size: &F) -> usize
|
||||||
|
where
|
||||||
|
F: Fn(&[T]) -> u64,
|
||||||
|
{
|
||||||
|
if serializables.is_empty() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let mut num = serializables.len();
|
||||||
|
let mut upper = serializables.len();
|
||||||
|
let mut lower = 1; // if one won't fit, we have a lot of TODOs
|
||||||
|
let mut next = serializables.len(); // optimistic
|
||||||
|
loop {
|
||||||
|
debug!(
|
||||||
|
"num {}, upper {} lower {} next {} serializables.len() {}",
|
||||||
|
num,
|
||||||
|
upper,
|
||||||
|
lower,
|
||||||
|
next,
|
||||||
|
serializables.len()
|
||||||
|
);
|
||||||
|
if serialized_size(&serializables[..num]) <= max_size {
|
||||||
|
next = (upper + num) / 2;
|
||||||
|
lower = num;
|
||||||
|
debug!("num {} fits, maybe too well? trying {}", num, next);
|
||||||
|
} else {
|
||||||
|
next = (lower + num) / 2;
|
||||||
|
upper = num;
|
||||||
|
debug!("num {} doesn't fit! trying {}", num, next);
|
||||||
|
}
|
||||||
|
// same as last time
|
||||||
|
if next == num {
|
||||||
|
debug!("converged on num {}", num);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
num = next;
|
||||||
|
}
|
||||||
|
num
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn split_serializable_chunks<T, R, F1, F2>(
|
||||||
|
serializables: &[T],
|
||||||
|
max_size: u64,
|
||||||
|
serialized_size: &F1,
|
||||||
|
converter: &mut F2,
|
||||||
|
) -> Vec<R>
|
||||||
|
where
|
||||||
|
F1: Fn(&[T]) -> u64,
|
||||||
|
F2: FnMut(&[T]) -> R,
|
||||||
|
{
|
||||||
|
let mut result = vec![];
|
||||||
|
let mut chunk_start = 0;
|
||||||
|
while chunk_start < serializables.len() {
|
||||||
|
let chunk_end =
|
||||||
|
chunk_start + num_will_fit(&serializables[chunk_start..], max_size, serialized_size);
|
||||||
|
result.push(converter(&serializables[chunk_start..chunk_end]));
|
||||||
|
chunk_start = chunk_end;
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
/// Creates the next entries for given transactions, outputs
|
/// Creates the next entries for given transactions, outputs
|
||||||
/// updates start_hash to hash of last Entry, sets num_hashes to 0
|
/// updates start_hash to hash of last Entry, sets num_hashes to 0
|
||||||
pub fn next_entries_mut(
|
pub fn next_entries_mut(
|
||||||
|
@ -310,61 +335,12 @@ pub fn next_entries_mut(
|
||||||
num_hashes: &mut u64,
|
num_hashes: &mut u64,
|
||||||
transactions: Vec<Transaction>,
|
transactions: Vec<Transaction>,
|
||||||
) -> Vec<Entry> {
|
) -> Vec<Entry> {
|
||||||
// TODO: ?? find a number that works better than |?
|
split_serializable_chunks(
|
||||||
// V
|
&transactions[..],
|
||||||
if transactions.is_empty() || transactions.len() == 1 {
|
BLOB_DATA_SIZE as u64,
|
||||||
vec![Entry::new_mut(start_hash, num_hashes, transactions)]
|
&Entry::serialized_size,
|
||||||
} else {
|
&mut |txs: &[Transaction]| Entry::new_mut(start_hash, num_hashes, txs.to_vec()),
|
||||||
let mut chunk_start = 0;
|
)
|
||||||
let mut entries = Vec::new();
|
|
||||||
|
|
||||||
while chunk_start < transactions.len() {
|
|
||||||
let mut chunk_end = transactions.len();
|
|
||||||
let mut upper = chunk_end;
|
|
||||||
let mut lower = chunk_start;
|
|
||||||
let mut next = chunk_end; // be optimistic that all will fit
|
|
||||||
|
|
||||||
// binary search for how many transactions will fit in an Entry (i.e. a BLOB)
|
|
||||||
loop {
|
|
||||||
debug!(
|
|
||||||
"chunk_end {}, upper {} lower {} next {} transactions.len() {}",
|
|
||||||
chunk_end,
|
|
||||||
upper,
|
|
||||||
lower,
|
|
||||||
next,
|
|
||||||
transactions.len()
|
|
||||||
);
|
|
||||||
if Entry::serialized_size(&transactions[chunk_start..chunk_end])
|
|
||||||
<= BLOB_DATA_SIZE as u64
|
|
||||||
{
|
|
||||||
next = (upper + chunk_end) / 2;
|
|
||||||
lower = chunk_end;
|
|
||||||
debug!(
|
|
||||||
"chunk_end {} fits, maybe too well? trying {}",
|
|
||||||
chunk_end, next
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
next = (lower + chunk_end) / 2;
|
|
||||||
upper = chunk_end;
|
|
||||||
debug!("chunk_end {} doesn't fit! trying {}", chunk_end, next);
|
|
||||||
}
|
|
||||||
// same as last time
|
|
||||||
if next == chunk_end {
|
|
||||||
debug!("converged on chunk_end {}", chunk_end);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
chunk_end = next;
|
|
||||||
}
|
|
||||||
entries.push(Entry::new_mut(
|
|
||||||
start_hash,
|
|
||||||
num_hashes,
|
|
||||||
transactions[chunk_start..chunk_end].to_vec(),
|
|
||||||
));
|
|
||||||
chunk_start = chunk_end;
|
|
||||||
}
|
|
||||||
|
|
||||||
entries
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Creates the next Entries for given transactions
|
/// Creates the next Entries for given transactions
|
||||||
|
@ -685,4 +661,40 @@ mod tests {
|
||||||
assert!(entries0.verify(&hash));
|
assert!(entries0.verify(&hash));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_num_will_fit_empty() {
|
||||||
|
let serializables: Vec<u32> = vec![];
|
||||||
|
let result = num_will_fit(&serializables[..], 8, &|_| 4);
|
||||||
|
assert_eq!(result, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_num_fit() {
|
||||||
|
let serializables_vec: Vec<u8> = (0..10).map(|_| 1).collect();
|
||||||
|
let serializables = &serializables_vec[..];
|
||||||
|
let sum = |i: &[u8]| (0..i.len()).into_iter().sum::<usize>() as u64;
|
||||||
|
// sum[0] is = 0, but sum[0..1] > 0, so result contains 1 item
|
||||||
|
let result = num_will_fit(serializables, 0, &sum);
|
||||||
|
assert_eq!(result, 1);
|
||||||
|
|
||||||
|
// sum[0..3] is <= 8, but sum[0..4] > 8, so result contains 3 items
|
||||||
|
let result = num_will_fit(serializables, 8, &sum);
|
||||||
|
assert_eq!(result, 4);
|
||||||
|
|
||||||
|
// sum[0..1] is = 1, but sum[0..2] > 0, so result contains 2 items
|
||||||
|
let result = num_will_fit(serializables, 1, &sum);
|
||||||
|
assert_eq!(result, 2);
|
||||||
|
|
||||||
|
// sum[0..9] = 45, so contains all items
|
||||||
|
let result = num_will_fit(serializables, 45, &sum);
|
||||||
|
assert_eq!(result, 10);
|
||||||
|
|
||||||
|
// sum[0..8] <= 44, but sum[0..9] = 45, so contains all but last item
|
||||||
|
let result = num_will_fit(serializables, 44, &sum);
|
||||||
|
assert_eq!(result, 9);
|
||||||
|
|
||||||
|
// sum[0..9] <= 46, but contains all items
|
||||||
|
let result = num_will_fit(serializables, 46, &sum);
|
||||||
|
assert_eq!(result, 10);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue