From 4a9d7318d112eb2de32dd42d6ef3095d29b4698f Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 13 Dec 2021 23:34:43 -0600 Subject: [PATCH] Rework test parameters to be shreds instead of entries (#21780) The number of shreds that result from a given number of entries is variable and in our test case, somewhat unintuitive to think about when trying to determine how much data we're pushing into the blockstore. So, this change converts the unit of test parameters from entries to shreds. This change also cleans up some variable naming for clarity and prints. --- core/tests/ledger_cleanup.rs | 86 ++++++++++++++++++------------------ ledger/src/blockstore.rs | 17 ++++++- 2 files changed, 60 insertions(+), 43 deletions(-) diff --git a/core/tests/ledger_cleanup.rs b/core/tests/ledger_cleanup.rs index c8e6c909c3..c55eaefa3c 100644 --- a/core/tests/ledger_cleanup.rs +++ b/core/tests/ledger_cleanup.rs @@ -7,7 +7,7 @@ mod tests { log::*, solana_core::ledger_cleanup_service::LedgerCleanupService, solana_ledger::{ - blockstore::{make_many_slot_entries, Blockstore}, + blockstore::{make_many_slot_shreds, Blockstore}, get_tmp_ledger_path, }, solana_measure::measure::Measure, @@ -26,9 +26,9 @@ mod tests { }; const DEFAULT_BENCHMARK_SLOTS: u64 = 50; - const DEFAULT_BATCH_SIZE: u64 = 1; + const DEFAULT_BATCH_SIZE_SLOTS: u64 = 1; const DEFAULT_MAX_LEDGER_SHREDS: u64 = 50; - const DEFAULT_ENTRIES_PER_SLOT: u64 = 500; + const DEFAULT_SHREDS_PER_SLOT: u64 = 25; const DEFAULT_STOP_SIZE_BYTES: u64 = 0; const DEFAULT_STOP_SIZE_ITERATIONS: u64 = 0; @@ -37,9 +37,9 @@ mod tests { #[derive(Debug)] struct BenchmarkConfig { benchmark_slots: u64, - batch_size: u64, + batch_size_slots: u64, max_ledger_shreds: u64, - entries_per_slot: u64, + shreds_per_slot: u64, stop_size_bytes: u64, stop_size_iterations: u64, pre_generate_data: bool, @@ -153,9 +153,9 @@ mod tests { fn get_benchmark_config() -> BenchmarkConfig { let benchmark_slots = read_env("BENCHMARK_SLOTS", DEFAULT_BENCHMARK_SLOTS); - let batch_size = read_env("BATCH_SIZE", DEFAULT_BATCH_SIZE); + let batch_size_slots = read_env("BATCH_SIZE", DEFAULT_BATCH_SIZE_SLOTS); let max_ledger_shreds = read_env("MAX_LEDGER_SHREDS", DEFAULT_MAX_LEDGER_SHREDS); - let entries_per_slot = read_env("ENTRIES_PER_SLOT", DEFAULT_ENTRIES_PER_SLOT); + let shreds_per_slot = read_env("SHREDS_PER_SLOT", DEFAULT_SHREDS_PER_SLOT); let stop_size_bytes = read_env("STOP_SIZE_BYTES", DEFAULT_STOP_SIZE_BYTES); let stop_size_iterations = read_env("STOP_SIZE_ITERATIONS", DEFAULT_STOP_SIZE_ITERATIONS); let pre_generate_data = read_env("PRE_GENERATE_DATA", false); @@ -171,9 +171,9 @@ mod tests { BenchmarkConfig { benchmark_slots, - batch_size, + batch_size_slots, max_ledger_shreds, - entries_per_slot, + shreds_per_slot, stop_size_bytes, stop_size_iterations, pre_generate_data, @@ -186,7 +186,7 @@ mod tests { } fn emit_header() { - println!("TIME_MS,DELTA_MS,START_SLOT,BATCH_SIZE,ENTRIES,MAX,SIZE,DELTA_SIZE,CPU_USER,CPU_SYSTEM,CPU_IDLE"); + println!("TIME_MS,DELTA_MS,START_SLOT,BATCH_SIZE,SHREDS,MAX,SIZE,DELTA_SIZE,CPU_USER,CPU_SYSTEM,CPU_IDLE"); } fn emit_stats( @@ -195,8 +195,8 @@ mod tests { storage_previous: &mut u64, start_slot: u64, batch_size: u64, - entries: u64, - max_slots: i64, + num_shreds: u64, + max_shreds: i64, blockstore: &Blockstore, cpu: &CpuStatsInner, ) { @@ -210,8 +210,8 @@ mod tests { time_now.duration_since(*time_previous).as_millis(), start_slot, batch_size, - entries, - max_slots, + num_shreds, + max_shreds, storage_now, storage_now as i64 - *storage_previous as i64, cpu_user, @@ -225,7 +225,7 @@ mod tests { /** * Example run command: - * BENCHMARK_SLOTS=10000 BATCH_SIZE=10 ENTRIES_PER_SLOT=1000 NUM_WRITERS=1 \ + * BENCHMARK_SLOTS=10000 BATCH_SIZE=10 SHREDS_PER_SLOT=1000 NUM_WRITERS=1 \ * PRE_GENERATE_DATA=true cargo test --release tests::test_ledger_cleanup_compaction \ * -- --exact --nocapture */ @@ -244,16 +244,16 @@ mod tests { eprintln!("LEDGER_PATH: {:?}", &blockstore_path); let benchmark_slots = config.benchmark_slots; - let batch_size = config.batch_size; + let batch_size_slots = config.batch_size_slots; let max_ledger_shreds = config.max_ledger_shreds; - let entries_per_slot = config.entries_per_slot; + let shreds_per_slot = config.shreds_per_slot; let stop_size_bytes = config.stop_size_bytes; let stop_size_iterations = config.stop_size_iterations; let pre_generate_data = config.pre_generate_data; let compaction_interval = config.compaction_interval; let num_writers = config.num_writers; - let batches = benchmark_slots / batch_size; + let num_batches = benchmark_slots / batch_size_slots; let (sender, receiver) = channel(); let exit = Arc::new(AtomicBool::new(false)); @@ -274,16 +274,25 @@ mod tests { if pre_generate_data { let t0 = Instant::now(); eprintln!("PRE_GENERATE_DATA: (this may take a while)"); - for i in 0..batches { - let start_slot = i * batch_size; + for i in 0..num_batches { + let start_slot = i * batch_size_slots; let (new_shreds, _) = - make_many_slot_entries(start_slot, batch_size, entries_per_slot); + make_many_slot_shreds(start_slot, batch_size_slots, shreds_per_slot); shreds.push_back(new_shreds); } eprintln!("PRE_GENERATE_DATA: took {} ms", t0.elapsed().as_millis()); } let shreds = Arc::new(Mutex::new(shreds)); + let (mut shreds_batch, entries) = + make_many_slot_shreds(0, batch_size_slots, shreds_per_slot); + info!( + "batch size (slots): {}, entries_per_slot: {}, shreds_per_slot: {}", + batch_size_slots, + entries.len() as u64 / batch_size_slots, + shreds_per_slot + ); + let time_initial = Instant::now(); let mut time_previous = time_initial; let mut storage_previous = 0; @@ -371,16 +380,8 @@ mod tests { insert_threads.push(insert_thread); } - let mut entries_batch = make_many_slot_entries(0, batch_size, entries_per_slot).0; - info!( - "batch size: {} entries_per_slot: {} shreds_per_slot: {}", - batch_size, - entries_per_slot, - entries_batch.len() - ); - - for i in 0..batches { - let start_slot = i * batch_size; + for i in 0..num_batches { + let start_slot = i * batch_size_slots; if time.elapsed().as_secs() > 0 { warn!( @@ -397,12 +398,12 @@ mod tests { if !pre_generate_data && shreds.lock().unwrap().len() < 50 { let mut make_time = Measure::start("make_entries"); - num_slots += batch_size; - total_slots += batch_size; - entries_batch + num_slots += batch_size_slots; + total_slots += batch_size_slots; + shreds_batch .iter_mut() - .for_each(|shred| shred.set_slot(shred.slot() + batch_size)); - let new_shreds = entries_batch.clone(); + .for_each(|shred| shred.set_slot(shred.slot() + batch_size_slots)); + let new_shreds = shreds_batch.clone(); shreds.lock().unwrap().push_back(new_shreds); make_time.stop(); total_make += make_time.as_us(); @@ -415,8 +416,8 @@ mod tests { &mut time_previous, &mut storage_previous, start_slot, - batch_size, - batch_size, + batch_size_slots, + shreds_per_slot, max_ledger_shreds as i64, &blockstore, &sys.get_stats(), @@ -525,12 +526,13 @@ mod tests { let blockstore = Arc::new(Blockstore::open(&blockstore_path).unwrap()); let n = 10_000; - let batch_size = 100; - let batches = n / batch_size; + let batch_size_slots = 100; + let num_batches = n / batch_size_slots; let max_ledger_shreds = 100; - for i in 0..batches { - let (shreds, _) = make_many_slot_entries(i * batch_size, batch_size, 1); + for i in 0..num_batches { + let start_slot = i * batch_size_slots; + let (shreds, _) = make_many_slot_shreds(start_slot, batch_size_slots, 1); blockstore.insert_shreds(shreds, None, false).unwrap(); } diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index e89e02091b..a7553f5a13 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -12,7 +12,10 @@ use { blockstore_meta::*, leader_schedule_cache::LeaderScheduleCache, next_slots_iterator::NextSlotsIterator, - shred::{Result as ShredResult, Shred, ShredType, Shredder, SHRED_PAYLOAD_SIZE}, + shred::{ + max_ticks_per_n_shreds, Result as ShredResult, Shred, ShredType, Shredder, + SHRED_PAYLOAD_SIZE, + }, }, bincode::deserialize, log::*, @@ -3824,6 +3827,18 @@ pub fn make_many_slot_entries( (shreds, entries) } +// used for tests only +// Create `num_shreds` shreds for [start_slot, start_slot + num_slot) slots +pub fn make_many_slot_shreds( + start_slot: u64, + num_slots: u64, + num_shreds_per_slot: u64, +) -> (Vec, Vec) { + // Use `None` as shred_size so the default (full) value is used + let num_entries = max_ticks_per_n_shreds(num_shreds_per_slot, None); + make_many_slot_entries(start_slot, num_slots, num_entries) +} + // Create shreds for slots that have a parent-child relationship defined by the input `chain` // used for tests only pub fn make_chaining_slot_entries(