(Ledger Store Benchmark) Add flags for checking shred storage size. (#22451)

core/tests/ledger_cleanup.rs

@@ -32,6 +32,7 @@ mod tests {
     const DEFAULT_SHREDS_PER_SLOT: u64 = 25;
     const DEFAULT_STOP_SIZE_BYTES: u64 = 0;
     const DEFAULT_STOP_SIZE_ITERATIONS: u64 = 0;
+    const DEFAULT_STOP_SIZE_CF_DATA_BYTES: u64 = 0;
     const DEFAULT_SHRED_DATA_CF_SIZE_BYTES: u64 = 125 * 1024 * 1024 * 1024;
 
     const ROCKSDB_FLUSH_GRACE_PERIOD_SECS: u64 = 20;
@@ -44,6 +45,7 @@ mod tests {
         shreds_per_slot: u64,
         stop_size_bytes: u64,
         stop_size_iterations: u64,
+        stop_size_cf_data_bytes: u64,
         pre_generate_data: bool,
         cleanup_blockstore: bool,
         assert_compaction: bool,
@@ -172,11 +174,14 @@ mod tests {
     /// Advanced benchmark settings:
     /// - `STOP_SIZE_BYTES`: if specified, the benchmark will count how
     ///   many times the ledger store size exceeds the specified threshold.
-    /// - `STOP_SIZE_ITERATIONS`: when `STOP_SIZE_BYTES` is specified, the
+    /// - `STOP_SIZE_CF_DATA_BYTES`: if specified, the benchmark will count how
-    ///   benchmark will stop immediately when the number of times where the
+    ///   many times the storage size of `cf::ShredData` which stores data shred
-    ///   ledger store size exceeds the configured `STOP_SIZE_BYTES`.  These
+    ///   exceeds the specified threshold.
-    ///   configs are used to make sure the benchmark runs successfully under
+    /// - `STOP_SIZE_ITERATIONS`: when any of the stop size is specified, the
-    ///   the storage limitation.
+    ///   benchmark will stop immediately when the number of consecutive times
+    ///   where the ledger store size exceeds the configured `STOP_SIZE_BYTES`.
+    ///   These configs are used to make sure the benchmark runs successfully
+    ///   under the storage limitation.
     /// - `CLEANUP_BLOCKSTORE`: if true, the ledger store created in the current
     ///   benchmark run will be deleted.  Default: true.
     /// - `NO_COMPACTION`: whether to stop rocksdb's background compaction
@@ -208,6 +213,8 @@ mod tests {
         let shreds_per_slot = read_env("SHREDS_PER_SLOT", DEFAULT_SHREDS_PER_SLOT);
         let stop_size_bytes = read_env("STOP_SIZE_BYTES", DEFAULT_STOP_SIZE_BYTES);
         let stop_size_iterations = read_env("STOP_SIZE_ITERATIONS", DEFAULT_STOP_SIZE_ITERATIONS);
+        let stop_size_cf_data_bytes =
+            read_env("STOP_SIZE_CF_DATA_BYTES", DEFAULT_STOP_SIZE_CF_DATA_BYTES);
         let pre_generate_data = read_env("PRE_GENERATE_DATA", false);
         let cleanup_blockstore = read_env("CLEANUP_BLOCKSTORE", true);
         // set default to `true` once compaction is merged
@@ -233,6 +240,7 @@ mod tests {
             shreds_per_slot,
             stop_size_bytes,
             stop_size_iterations,
+            stop_size_cf_data_bytes,
             pre_generate_data,
             cleanup_blockstore,
             assert_compaction,
@@ -289,6 +297,37 @@ mod tests {
         *data_shred_storage_previous = data_shred_storage_now;
     }
 
+    /// Helper function of the benchmark `test_ledger_cleanup_compaction` which
+    /// returns true if the benchmark fails the size limitation check.
+    fn is_exceeded_stop_size_iterations(
+        storage_size: u64,
+        stop_size: u64,
+        exceeded_iterations: &mut u64,
+        iteration_limit: u64,
+        storage_desc: &str,
+    ) -> bool {
+        if stop_size > 0 {
+            if storage_size >= stop_size {
+                *exceeded_iterations += 1;
+                warn!(
+                    "{} size {} exceeds the stop size {} for {} times!",
+                    storage_desc, storage_size, stop_size, exceeded_iterations
+                );
+            } else {
+                *exceeded_iterations = 0;
+            }
+
+            if *exceeded_iterations >= iteration_limit {
+                error!(
+                    "{} size exceeds the configured limit {} for {} times",
+                    storage_desc, stop_size, exceeded_iterations,
+                );
+                return true;
+            }
+        }
+        false
+    }
+
     /// The ledger cleanup compaction test which can also be used as a benchmark
     /// measuring shred insertion performance of the blockstore.
     ///
@@ -332,6 +371,7 @@ mod tests {
         let shreds_per_slot = config.shreds_per_slot;
         let stop_size_bytes = config.stop_size_bytes;
         let stop_size_iterations = config.stop_size_iterations;
+        let stop_size_cf_data_bytes = config.stop_size_cf_data_bytes;
         let pre_generate_data = config.pre_generate_data;
         let compaction_interval = config.compaction_interval;
         let num_writers = config.num_writers;
@@ -388,6 +428,7 @@ mod tests {
         let mut storage_previous = 0;
         let mut data_shred_storage_previous = 0;
         let mut stop_size_bytes_exceeded_iterations = 0;
+        let mut stop_size_cf_data_exceeded_iterations = 0;
 
         emit_header();
         emit_stats(
@@ -535,16 +576,24 @@ mod tests {
                 &sys.get_stats(),
             );
 
-            if stop_size_bytes > 0 {
+            if is_exceeded_stop_size_iterations(
-                if storage_previous >= stop_size_bytes {
+                storage_previous,
-                    stop_size_bytes_exceeded_iterations += 1;
+                stop_size_bytes,
-                } else {
+                &mut stop_size_bytes_exceeded_iterations,
-                    stop_size_bytes_exceeded_iterations = 0;
+                stop_size_iterations,
-                }
+                "Storage",
-
+            ) {
-                if stop_size_bytes_exceeded_iterations > stop_size_iterations {
                 break;
             }
+
+            if is_exceeded_stop_size_iterations(
+                data_shred_storage_previous,
+                stop_size_cf_data_bytes,
+                &mut stop_size_cf_data_exceeded_iterations,
+                stop_size_iterations,
+                "cf::ShredData",
+            ) {
+                break;
             }
 
             if finished_batch >= num_batches {