cleanup: do not count rows unless enabled by argument "--count-rows" (#63)

cleanup: do not count rows unless enabled by argument "--count-rows"

.github/workflows/rust_check.yml

@@ -0,0 +1,51 @@
+name: Cargo Build & Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  CARGO_TERM_COLOR: always
+  SCCACHE_GHA_ENABLED: true
+  RUSTC_WRAPPER: sccache
+  SCCACHE_CACHE_SIZE: "1G"
+
+jobs:
+  build_and_test:
+    name: bankingstage-sidecar check and test
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Install Linux Packages
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install libssl-dev openssl -y
+
+      - uses: actions/checkout@v4
+
+      # The toolchain action should definitely be run before the cache action
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache: true
+          # avoid the default "-D warnings" which thrashes cache
+          rustflags: ""
+
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.3
+
+      # https://github.com/actions/cache/blob/main/examples.md#rust---cargo
+      # https://blog.arriven.wtf/posts/rust-ci-cache/
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # will be covered by sscache
+          cache-targets: false
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+
+      - name: Early Build
+        run: |
+          cargo check --locked --workspace --all-targets
+
+      - name: Run Tests
+        run: RUST_LOG=info cargo test

run-service-and-cleanup.sh

@@ -11,6 +11,7 @@ while true; do
   # startup delay
   sleep 300;
   # should retain about 1 week of data
+  # no "--count-rows"
   RUST_LOG=info /usr/local/bin/cleanupdb --num-slots-to-keep $SLOTS_TO_KEEP;
   # every 5 hours
   sleep 18000;

src/bin/cleanupdb.rs

@@ -13,6 +13,9 @@ pub struct Args {
 
     #[arg(short, long, default_value_t = false)]
     pub dry_run: bool,
+
+    #[arg(short, long, default_value_t = false)]
+    pub count_rows: bool,
 }
 
 #[tokio::main()]
@@ -22,9 +25,10 @@ async fn main() {
     let Args {
         num_slots_to_keep,
         dry_run,
+        count_rows,
     } = Args::parse();
 
     let session = PostgresSession::new(0).await.unwrap();
 
-    session.cleanup_old_data(num_slots_to_keep, dry_run).await;
+    session.cleanup_old_data(num_slots_to_keep, dry_run, count_rows).await;
 }

src/postgres.rs

@@ -65,7 +65,7 @@ lazy_static::lazy_static! {
 
     static ref TIME_TO_STORE_TX_ACCOUNT_OLD: IntGauge =
        register_int_gauge!(opts!("banking_stage_sidecar_tx_account_old", "Account in tx account old")).unwrap();
-    
+
     static ref TIME_TO_STORE_TX_ACCOUNT_NEW: IntGauge =
        register_int_gauge!(opts!("banking_stage_sidecar_tx_account_new", "Account in tx account new")).unwrap();
 }
@@ -413,7 +413,6 @@ impl PostgresSession {
         &self,
         accounts_for_transaction: Vec<AccountsForTransaction>,
     ) -> anyhow::Result<()> {
-
         let instant = Instant::now();
         let temp_table = self.get_new_temp_table();
         self.client
@@ -959,7 +958,9 @@ impl PostgresSession {
     // "accounts_map_transaction" -> delete rows with transaction_id before X
     // "transaction_infos" -> delete rows processed_slot before X
     // "transaction_slot" -> delete transaction with slot before X
-    pub async fn cleanup_old_data(&self, slots_to_keep: i64, dry_run: bool) {
+
+    // count_rows=true might be very expensive
+    pub async fn cleanup_old_data(&self, slots_to_keep: i64, dry_run: bool, count_rows: bool) {
         // keep 1mio slots (apprx 4 days)
         info!(
             "{}Running cleanup job with slots_to_keep={}",
@@ -969,7 +970,7 @@ impl PostgresSession {
 
         self.configure_work_mem().await;
 
-        {
+        if count_rows {
             info!("Rows before cleanup:");
             self.log_rowcount(Level::Info, "blocks").await;
             self.log_rowcount(Level::Info, "accounts").await;
@@ -979,6 +980,8 @@ impl PostgresSession {
                 .await;
             self.log_rowcount(Level::Info, "transaction_infos").await;
             self.log_rowcount(Level::Info, "transaction_slot").await;
+        } else {
+            info!("Skipping row count before cleanup");
         }
 
         // max slot from blocks table
@@ -1227,7 +1230,7 @@ impl PostgresSession {
             );
         }
 
-        {
+        if count_rows {
             info!("Rows after cleanup:");
             self.log_rowcount(Level::Info, "blocks").await;
             self.log_rowcount(Level::Info, "accounts").await;
@@ -1237,6 +1240,8 @@ impl PostgresSession {
                 .await;
             self.log_rowcount(Level::Info, "transaction_infos").await;
             self.log_rowcount(Level::Info, "transaction_slot").await;
+        } else {
+            info!("Skipping row count after cleanup");
         }
 
         info!("Cleanup job completed.");