cleanup: do not count rows unless enabled by argument "--count-rows" (#63)
cleanup: do not count rows unless enabled by argument "--count-rows"
This commit is contained in:
parent
1515ee2119
commit
78344dc8b4
|
@ -0,0 +1,51 @@
|
||||||
|
name: Cargo Build & Test
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
env:
|
||||||
|
CARGO_TERM_COLOR: always
|
||||||
|
SCCACHE_GHA_ENABLED: true
|
||||||
|
RUSTC_WRAPPER: sccache
|
||||||
|
SCCACHE_CACHE_SIZE: "1G"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build_and_test:
|
||||||
|
name: bankingstage-sidecar check and test
|
||||||
|
runs-on: ubuntu-22.04
|
||||||
|
steps:
|
||||||
|
- name: Install Linux Packages
|
||||||
|
run: |
|
||||||
|
sudo apt-get update -y
|
||||||
|
sudo apt-get install libssl-dev openssl -y
|
||||||
|
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
# The toolchain action should definitely be run before the cache action
|
||||||
|
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||||
|
with:
|
||||||
|
cache: true
|
||||||
|
# avoid the default "-D warnings" which thrashes cache
|
||||||
|
rustflags: ""
|
||||||
|
|
||||||
|
- name: Run sccache-cache
|
||||||
|
uses: mozilla-actions/sccache-action@v0.0.3
|
||||||
|
|
||||||
|
# https://github.com/actions/cache/blob/main/examples.md#rust---cargo
|
||||||
|
# https://blog.arriven.wtf/posts/rust-ci-cache/
|
||||||
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
with:
|
||||||
|
# will be covered by sscache
|
||||||
|
cache-targets: false
|
||||||
|
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
|
||||||
|
|
||||||
|
|
||||||
|
- name: Early Build
|
||||||
|
run: |
|
||||||
|
cargo check --locked --workspace --all-targets
|
||||||
|
|
||||||
|
- name: Run Tests
|
||||||
|
run: RUST_LOG=info cargo test
|
|
@ -11,6 +11,7 @@ while true; do
|
||||||
# startup delay
|
# startup delay
|
||||||
sleep 300;
|
sleep 300;
|
||||||
# should retain about 1 week of data
|
# should retain about 1 week of data
|
||||||
|
# no "--count-rows"
|
||||||
RUST_LOG=info /usr/local/bin/cleanupdb --num-slots-to-keep $SLOTS_TO_KEEP;
|
RUST_LOG=info /usr/local/bin/cleanupdb --num-slots-to-keep $SLOTS_TO_KEEP;
|
||||||
# every 5 hours
|
# every 5 hours
|
||||||
sleep 18000;
|
sleep 18000;
|
||||||
|
|
|
@ -13,6 +13,9 @@ pub struct Args {
|
||||||
|
|
||||||
#[arg(short, long, default_value_t = false)]
|
#[arg(short, long, default_value_t = false)]
|
||||||
pub dry_run: bool,
|
pub dry_run: bool,
|
||||||
|
|
||||||
|
#[arg(short, long, default_value_t = false)]
|
||||||
|
pub count_rows: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main()]
|
#[tokio::main()]
|
||||||
|
@ -22,9 +25,10 @@ async fn main() {
|
||||||
let Args {
|
let Args {
|
||||||
num_slots_to_keep,
|
num_slots_to_keep,
|
||||||
dry_run,
|
dry_run,
|
||||||
|
count_rows,
|
||||||
} = Args::parse();
|
} = Args::parse();
|
||||||
|
|
||||||
let session = PostgresSession::new(0).await.unwrap();
|
let session = PostgresSession::new(0).await.unwrap();
|
||||||
|
|
||||||
session.cleanup_old_data(num_slots_to_keep, dry_run).await;
|
session.cleanup_old_data(num_slots_to_keep, dry_run, count_rows).await;
|
||||||
}
|
}
|
||||||
|
|
|
@ -65,7 +65,7 @@ lazy_static::lazy_static! {
|
||||||
|
|
||||||
static ref TIME_TO_STORE_TX_ACCOUNT_OLD: IntGauge =
|
static ref TIME_TO_STORE_TX_ACCOUNT_OLD: IntGauge =
|
||||||
register_int_gauge!(opts!("banking_stage_sidecar_tx_account_old", "Account in tx account old")).unwrap();
|
register_int_gauge!(opts!("banking_stage_sidecar_tx_account_old", "Account in tx account old")).unwrap();
|
||||||
|
|
||||||
static ref TIME_TO_STORE_TX_ACCOUNT_NEW: IntGauge =
|
static ref TIME_TO_STORE_TX_ACCOUNT_NEW: IntGauge =
|
||||||
register_int_gauge!(opts!("banking_stage_sidecar_tx_account_new", "Account in tx account new")).unwrap();
|
register_int_gauge!(opts!("banking_stage_sidecar_tx_account_new", "Account in tx account new")).unwrap();
|
||||||
}
|
}
|
||||||
|
@ -413,7 +413,6 @@ impl PostgresSession {
|
||||||
&self,
|
&self,
|
||||||
accounts_for_transaction: Vec<AccountsForTransaction>,
|
accounts_for_transaction: Vec<AccountsForTransaction>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
|
|
||||||
let instant = Instant::now();
|
let instant = Instant::now();
|
||||||
let temp_table = self.get_new_temp_table();
|
let temp_table = self.get_new_temp_table();
|
||||||
self.client
|
self.client
|
||||||
|
@ -959,7 +958,9 @@ impl PostgresSession {
|
||||||
// "accounts_map_transaction" -> delete rows with transaction_id before X
|
// "accounts_map_transaction" -> delete rows with transaction_id before X
|
||||||
// "transaction_infos" -> delete rows processed_slot before X
|
// "transaction_infos" -> delete rows processed_slot before X
|
||||||
// "transaction_slot" -> delete transaction with slot before X
|
// "transaction_slot" -> delete transaction with slot before X
|
||||||
pub async fn cleanup_old_data(&self, slots_to_keep: i64, dry_run: bool) {
|
|
||||||
|
// count_rows=true might be very expensive
|
||||||
|
pub async fn cleanup_old_data(&self, slots_to_keep: i64, dry_run: bool, count_rows: bool) {
|
||||||
// keep 1mio slots (apprx 4 days)
|
// keep 1mio slots (apprx 4 days)
|
||||||
info!(
|
info!(
|
||||||
"{}Running cleanup job with slots_to_keep={}",
|
"{}Running cleanup job with slots_to_keep={}",
|
||||||
|
@ -969,7 +970,7 @@ impl PostgresSession {
|
||||||
|
|
||||||
self.configure_work_mem().await;
|
self.configure_work_mem().await;
|
||||||
|
|
||||||
{
|
if count_rows {
|
||||||
info!("Rows before cleanup:");
|
info!("Rows before cleanup:");
|
||||||
self.log_rowcount(Level::Info, "blocks").await;
|
self.log_rowcount(Level::Info, "blocks").await;
|
||||||
self.log_rowcount(Level::Info, "accounts").await;
|
self.log_rowcount(Level::Info, "accounts").await;
|
||||||
|
@ -979,6 +980,8 @@ impl PostgresSession {
|
||||||
.await;
|
.await;
|
||||||
self.log_rowcount(Level::Info, "transaction_infos").await;
|
self.log_rowcount(Level::Info, "transaction_infos").await;
|
||||||
self.log_rowcount(Level::Info, "transaction_slot").await;
|
self.log_rowcount(Level::Info, "transaction_slot").await;
|
||||||
|
} else {
|
||||||
|
info!("Skipping row count before cleanup");
|
||||||
}
|
}
|
||||||
|
|
||||||
// max slot from blocks table
|
// max slot from blocks table
|
||||||
|
@ -1227,7 +1230,7 @@ impl PostgresSession {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
if count_rows {
|
||||||
info!("Rows after cleanup:");
|
info!("Rows after cleanup:");
|
||||||
self.log_rowcount(Level::Info, "blocks").await;
|
self.log_rowcount(Level::Info, "blocks").await;
|
||||||
self.log_rowcount(Level::Info, "accounts").await;
|
self.log_rowcount(Level::Info, "accounts").await;
|
||||||
|
@ -1237,6 +1240,8 @@ impl PostgresSession {
|
||||||
.await;
|
.await;
|
||||||
self.log_rowcount(Level::Info, "transaction_infos").await;
|
self.log_rowcount(Level::Info, "transaction_infos").await;
|
||||||
self.log_rowcount(Level::Info, "transaction_slot").await;
|
self.log_rowcount(Level::Info, "transaction_slot").await;
|
||||||
|
} else {
|
||||||
|
info!("Skipping row count after cleanup");
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Cleanup job completed.");
|
info!("Cleanup job completed.");
|
||||||
|
|
Loading…
Reference in New Issue