diff --git a/.github/workflows/rust_check.yml b/.github/workflows/rust_check.yml new file mode 100644 index 0000000..7a52018 --- /dev/null +++ b/.github/workflows/rust_check.yml @@ -0,0 +1,51 @@ +name: Cargo Build & Test + +on: + push: + branches: + - main + pull_request: + +env: + CARGO_TERM_COLOR: always + SCCACHE_GHA_ENABLED: true + RUSTC_WRAPPER: sccache + SCCACHE_CACHE_SIZE: "1G" + +jobs: + build_and_test: + name: bankingstage-sidecar check and test + runs-on: ubuntu-22.04 + steps: + - name: Install Linux Packages + run: | + sudo apt-get update -y + sudo apt-get install libssl-dev openssl -y + + - uses: actions/checkout@v4 + + # The toolchain action should definitely be run before the cache action + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: true + # avoid the default "-D warnings" which thrashes cache + rustflags: "" + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + # https://github.com/actions/cache/blob/main/examples.md#rust---cargo + # https://blog.arriven.wtf/posts/rust-ci-cache/ + - uses: Swatinem/rust-cache@v2 + with: + # will be covered by sscache + cache-targets: false + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + + - name: Early Build + run: | + cargo check --locked --workspace --all-targets + + - name: Run Tests + run: RUST_LOG=info cargo test diff --git a/run-service-and-cleanup.sh b/run-service-and-cleanup.sh index ffaebb6..1c3a638 100755 --- a/run-service-and-cleanup.sh +++ b/run-service-and-cleanup.sh @@ -11,6 +11,7 @@ while true; do # startup delay sleep 300; # should retain about 1 week of data + # no "--count-rows" RUST_LOG=info /usr/local/bin/cleanupdb --num-slots-to-keep $SLOTS_TO_KEEP; # every 5 hours sleep 18000; diff --git a/src/bin/cleanupdb.rs b/src/bin/cleanupdb.rs index 325f710..10a69f1 100644 --- a/src/bin/cleanupdb.rs +++ b/src/bin/cleanupdb.rs @@ -13,6 +13,9 @@ pub struct Args { #[arg(short, long, default_value_t = false)] pub dry_run: bool, + + #[arg(short, long, default_value_t = false)] + pub count_rows: bool, } #[tokio::main()] @@ -22,9 +25,10 @@ async fn main() { let Args { num_slots_to_keep, dry_run, + count_rows, } = Args::parse(); let session = PostgresSession::new(0).await.unwrap(); - session.cleanup_old_data(num_slots_to_keep, dry_run).await; + session.cleanup_old_data(num_slots_to_keep, dry_run, count_rows).await; } diff --git a/src/postgres.rs b/src/postgres.rs index 49a27f0..58353c8 100644 --- a/src/postgres.rs +++ b/src/postgres.rs @@ -65,7 +65,7 @@ lazy_static::lazy_static! { static ref TIME_TO_STORE_TX_ACCOUNT_OLD: IntGauge = register_int_gauge!(opts!("banking_stage_sidecar_tx_account_old", "Account in tx account old")).unwrap(); - + static ref TIME_TO_STORE_TX_ACCOUNT_NEW: IntGauge = register_int_gauge!(opts!("banking_stage_sidecar_tx_account_new", "Account in tx account new")).unwrap(); } @@ -413,7 +413,6 @@ impl PostgresSession { &self, accounts_for_transaction: Vec, ) -> anyhow::Result<()> { - let instant = Instant::now(); let temp_table = self.get_new_temp_table(); self.client @@ -959,7 +958,9 @@ impl PostgresSession { // "accounts_map_transaction" -> delete rows with transaction_id before X // "transaction_infos" -> delete rows processed_slot before X // "transaction_slot" -> delete transaction with slot before X - pub async fn cleanup_old_data(&self, slots_to_keep: i64, dry_run: bool) { + + // count_rows=true might be very expensive + pub async fn cleanup_old_data(&self, slots_to_keep: i64, dry_run: bool, count_rows: bool) { // keep 1mio slots (apprx 4 days) info!( "{}Running cleanup job with slots_to_keep={}", @@ -969,7 +970,7 @@ impl PostgresSession { self.configure_work_mem().await; - { + if count_rows { info!("Rows before cleanup:"); self.log_rowcount(Level::Info, "blocks").await; self.log_rowcount(Level::Info, "accounts").await; @@ -979,6 +980,8 @@ impl PostgresSession { .await; self.log_rowcount(Level::Info, "transaction_infos").await; self.log_rowcount(Level::Info, "transaction_slot").await; + } else { + info!("Skipping row count before cleanup"); } // max slot from blocks table @@ -1227,7 +1230,7 @@ impl PostgresSession { ); } - { + if count_rows { info!("Rows after cleanup:"); self.log_rowcount(Level::Info, "blocks").await; self.log_rowcount(Level::Info, "accounts").await; @@ -1237,6 +1240,8 @@ impl PostgresSession { .await; self.log_rowcount(Level::Info, "transaction_infos").await; self.log_rowcount(Level::Info, "transaction_slot").await; + } else { + info!("Skipping row count after cleanup"); } info!("Cleanup job completed.");