2023-12-12 15:51:28 -08:00
# This workflow is designed to delete old Google Cloud Platform (GCP) resources to save on costs.
#
# 1. Deletes specific instances in GCP older than a defined number of days.
# 2. Deletes instance templates older than a set number of days.
# 3. Deletes older disks not currently in use, with certain ones prefixed by commit hashes or "zebrad-".
# 4. Deletes cache images from GCP, retaining a specified number of the latest images for certain types like zebrad checkpoint cache, zebrad tip cache, and lightwalletd + zebrad tip cache.
# 5. Deletes unused artifacts from Google Artifact Registry older than a defined number of hours while retaining the latest few.
#
# It uses the gcloud CLI for most of its operations and also leverages specific GitHub Actions like the gcr-cleaner for deleting old images from the Google Artifact Registry.
# The workflow is scheduled to run daily at 0700 UTC.
2022-06-15 14:59:55 -07:00
name : Delete GCP resources
on :
2022-09-08 13:24:24 -07:00
# Run daily, when most devs aren't working
# 0700 UTC is after AEST working hours but before ET working hours
2022-06-15 14:59:55 -07:00
schedule :
2022-09-08 13:24:24 -07:00
- cron : "0 7 * * *"
2022-06-15 14:59:55 -07:00
workflow_dispatch :
2022-09-05 19:51:46 -07:00
env :
2022-09-21 16:16:38 -07:00
# Delete all resources created before $DELETE_INSTANCE_DAYS days ago.
# We keep this short to reduce CPU, RAM, and storage costs.
DELETE_INSTANCE_DAYS : 3
# Delete all other resources created before $DELETE_AGE_DAYS days ago.
2022-09-08 13:24:24 -07:00
# We keep this short to reduce storage costs.
DELETE_AGE_DAYS : 2
2022-09-05 19:51:46 -07:00
# But keep the latest $KEEP_LATEST_IMAGE_COUNT images of each type.
2022-09-08 13:24:24 -07:00
# We keep this small to reduce storage costs.
KEEP_LATEST_IMAGE_COUNT : 2
2023-01-16 15:38:14 -08:00
# Delete all artifacts in registry created before $DELETE_IMAGE_HOURS hours ago.
# We keep this long enough for PRs that are still on the same commit can re-run with the same image.
DELETE_IMAGE_HOURS : 504h # 21 days
2022-09-05 19:51:46 -07:00
2022-06-15 14:59:55 -07:00
jobs :
delete-resources :
name : Delete old GCP resources
runs-on : ubuntu-latest
permissions :
contents : 'read'
id-token : 'write'
steps :
2023-10-19 10:05:38 -07:00
- uses : actions/checkout@v4.1.1
2022-09-05 19:51:46 -07:00
with :
persist-credentials : false
2022-06-15 14:59:55 -07:00
# Setup gcloud CLI
- name : Authenticate to Google Cloud
id : auth
2023-12-04 17:57:57 -08:00
uses : google-github-actions/auth@v2.0.0
2022-06-15 14:59:55 -07:00
with :
2022-08-23 20:49:55 -07:00
retries : '3'
2023-04-12 23:56:21 -07:00
workload_identity_provider : '${{ vars.GCP_WIF }}'
service_account : '${{ vars.GCP_DEPLOYMENTS_SA }}'
2022-11-09 22:32:21 -08:00
- name : Set up Cloud SDK
2023-12-30 05:02:56 -08:00
uses : google-github-actions/setup-gcloud@v2.0.1
2022-06-15 14:59:55 -07:00
2023-04-27 19:26:29 -07:00
# Deletes all mainnet and testnet instances older than $DELETE_INSTANCE_DAYS days.
2022-09-21 16:16:38 -07:00
#
# We only delete instances that end in 7 or more hex characters,
# to avoid deleting managed instance groups and manually created instances.
#
# ${INSTANCE_AND_ZONE} expands to:
# <instance-name> --zone=<zone-name>
# so it can't be shell-quoted.
- name : Delete old instances
run : |
2023-12-12 15:51:28 -08:00
./.github/workflows/scripts/gcp-delete-old-instances.sh
2022-09-05 19:51:46 -07:00
# Deletes all the instance templates older than $DELETE_AGE_DAYS days.
2022-06-15 14:59:55 -07:00
- name : Delete old instance templates
run : |
2023-12-12 15:51:28 -08:00
./.github/workflows/scripts/gcp-delete-old-templates.sh
2022-09-05 15:13:39 -07:00
2023-04-27 19:26:29 -07:00
# Deletes all mainnet and testnet disks older than $DELETE_AGE_DAYS days.
2022-09-05 15:13:39 -07:00
#
2022-09-21 16:16:38 -07:00
# Disks that are attached to an instance template can't be deleted, so it is safe to try to delete all disks here.
#
# ${DISK_AND_LOCATION} expands to:
# <disk-name> --[zone|region]=<location-name>
# so it can't be shell-quoted.
2022-09-05 19:51:46 -07:00
- name : Delete old disks
2022-09-05 15:13:39 -07:00
run : |
2023-12-12 15:51:28 -08:00
./.github/workflows/scripts/gcp-delete-old-disks.sh
2022-09-05 15:13:39 -07:00
2023-04-27 19:26:29 -07:00
# Deletes mainnet and testnet cache images older than $DELETE_AGE_DAYS days.
2022-09-05 19:51:46 -07:00
#
2022-09-13 00:24:25 -07:00
# Keeps all images younger than $DELETE_AGE_DAYS.
2023-04-27 19:26:29 -07:00
# Also keeps $KEEP_LATEST_IMAGE_COUNT older images of each type, for each network:
2022-09-05 19:51:46 -07:00
# - zebrad checkpoint cache
# - zebrad tip cache
# - lightwalletd + zebrad tip cache
#
2022-09-13 00:24:25 -07:00
# TODO:
2023-04-27 19:26:29 -07:00
# - refactor out repeated shell script code
2023-12-12 15:51:28 -08:00
- name : Delete old cache images
2022-09-05 19:51:46 -07:00
run : |
2023-12-12 15:51:28 -08:00
./.github/workflows/scripts/gcp-delete-old-cache-images.sh
2023-01-16 15:38:14 -08:00
# We're using a generic approach here, which allows multiple registries to be included,
# even those not related to GCP. Enough reason to create a separate job.
2023-04-27 19:26:29 -07:00
#
# The same artifacts are used for both mainnet and testnet.
2023-01-16 15:38:14 -08:00
clean-registries :
name : Delete unused artifacts in registry
runs-on : ubuntu-latest
permissions :
contents : 'read'
id-token : 'write'
steps :
2023-10-19 10:05:38 -07:00
- uses : actions/checkout@v4.1.1
2023-01-16 15:38:14 -08:00
with :
persist-credentials : false
# Setup gcloud CLI
- name : Authenticate to Google Cloud
id : auth
2023-12-04 17:57:57 -08:00
uses : google-github-actions/auth@v2.0.0
2023-01-16 15:38:14 -08:00
with :
retries : '3'
2023-04-12 23:56:21 -07:00
workload_identity_provider : '${{ vars.GCP_WIF }}'
service_account : '${{ vars.GCP_DEPLOYMENTS_SA }}'
2023-01-16 15:38:14 -08:00
token_format : 'access_token'
- name : Login to Google Artifact Registry
2023-09-14 11:07:31 -07:00
uses : docker/login-action@v3.0.0
2023-01-16 15:38:14 -08:00
with :
registry : us-docker.pkg.dev
username : oauth2accesstoken
password : ${{ steps.auth.outputs.access_token }}
# Deletes all images older than $DELETE_IMAGE_HOURS days.
- uses : 'docker://us-docker.pkg.dev/gcr-cleaner/gcr-cleaner/gcr-cleaner-cli'
continue-on-error: true # TODO : remove after fixig https://github.com/ZcashFoundation/zebra/issues/5933
# Refer to the official documentation to understand available arguments:
# https://github.com/GoogleCloudPlatform/gcr-cleaner
with :
args : >-
2023-04-12 23:56:21 -07:00
-repo=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra/zebrad-test
-repo=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra/lightwalletd
2023-01-16 15:38:14 -08:00
-grace=${{ env.DELETE_IMAGE_HOURS }}
-keep=${{ env.KEEP_LATEST_IMAGE_COUNT }}