bypasses rayon thread-pool for single entry batches (#28077)

With no parallelization, thread-pool only adds overhead.
This commit is contained in:
behzad nouri 2022-09-26 21:32:58 +00:00 committed by GitHub
parent b9849179c9
commit 72537e7e07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 49 additions and 11 deletions

View File

@ -2793,14 +2793,28 @@ impl Blockstore {
.map(|(_, end_index)| u64::from(*end_index) - start_index + 1)
.unwrap_or(0);
let entries: Result<Vec<Vec<Entry>>> = PAR_THREAD_POOL.install(|| {
let entries: Result<Vec<Vec<Entry>>> = if completed_ranges.len() <= 1 {
completed_ranges
.par_iter()
.into_iter()
.map(|(start_index, end_index)| {
self.get_entries_in_data_block(slot, *start_index, *end_index, Some(&slot_meta))
self.get_entries_in_data_block(slot, start_index, end_index, Some(&slot_meta))
})
.collect()
});
} else {
PAR_THREAD_POOL.install(|| {
completed_ranges
.into_par_iter()
.map(|(start_index, end_index)| {
self.get_entries_in_data_block(
slot,
start_index,
end_index,
Some(&slot_meta),
)
})
.collect()
})
};
let entries: Vec<Entry> = entries?.into_iter().flatten().collect();
Ok((entries, num_shreds, slot_meta.is_full()))
}

View File

@ -931,15 +931,25 @@ pub(super) fn make_shreds_from_data(
.collect();
// Generate coding shreds, populate merkle branch
// for all shreds and attach signature.
let shreds = thread_pool.install(|| {
let shreds: Result<Vec<_>, Error> = if shreds.len() <= 1 {
shreds
.into_par_iter()
.into_iter()
.zip(next_code_index)
.map(|(shreds, next_code_index)| {
make_erasure_batch(keypair, shreds, next_code_index, reed_solomon_cache)
})
.collect::<Result<Vec<_>, Error>>()
});
.collect()
} else {
thread_pool.install(|| {
shreds
.into_par_iter()
.zip(next_code_index)
.map(|(shreds, next_code_index)| {
make_erasure_batch(keypair, shreds, next_code_index, reed_solomon_cache)
})
.collect()
})
};
stats.gen_coding_elapsed += now.elapsed().as_micros() as u64;
shreds
}

View File

@ -216,15 +216,29 @@ impl Shredder {
)
.collect();
// 1) Generate coding shreds
let mut coding_shreds: Vec<_> = PAR_THREAD_POOL.install(|| {
let mut coding_shreds: Vec<_> = if chunks.len() <= 1 {
chunks
.into_par_iter()
.into_iter()
.zip(next_code_index)
.flat_map(|(shreds, next_code_index)| {
Shredder::generate_coding_shreds(&shreds, next_code_index, reed_solomon_cache)
})
.collect()
});
} else {
PAR_THREAD_POOL.install(|| {
chunks
.into_par_iter()
.zip(next_code_index)
.flat_map(|(shreds, next_code_index)| {
Shredder::generate_coding_shreds(
&shreds,
next_code_index,
reed_solomon_cache,
)
})
.collect()
})
};
gen_coding_time.stop();
let mut sign_coding_time = Measure::start("sign_coding_shreds");