zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

change(ci): Split checking for cached state disks into its own workflow (#6576) 

* Split checking for cached state disks into its own workflow

* Fix workflow field order

* Run the top-level workflow when the reusable workflow changes

* And run dependent workflows for pull requests as well

* Remove redundant output names

* Document the existing and new workflow jobs

* Add the network to the "no disk found" message
This commit is contained in:
teor 2023-05-03 22:04:16 +10:00 committed by GitHub
parent 9416043a80
commit aea3833c07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 127 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
.github/workflows/continous-integration-docker.yml vendored
View File

@ -53,6 +53,7 @@ on:
- '.github/workflows/continous-integration-docker.yml'
- '.github/workflows/deploy-gcp-tests.yml'
- '.github/workflows/build-docker-image.yml'
- '.github/workflows/find-cached-disks.yml'
push:
branches:
@ -75,87 +76,37 @@ on:
- '.github/workflows/continous-integration-docker.yml'
- '.github/workflows/deploy-gcp-tests.yml'
- '.github/workflows/build-docker-image.yml'
- '.github/workflows/find-cached-disks.yml'
jobs:
# Check if the cached state disks used by the tests are available for the default network.
#
# The default network is Mainnet unless a manually triggered workflow or repository variable
# is configured differently.
#
# The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml
get-available-disks:
runs-on: ubuntu-latest
name: Find available cached state disks
outputs:
lwd_tip_disk: ${{ steps.get-available-disks.outputs.lwd_tip_disk }}
zebra_tip_disk: ${{ steps.get-available-disks.outputs.zebra_tip_disk }}
zebra_checkpoint_disk: ${{ steps.get-available-disks.outputs.zebra_checkpoint_disk }}
permissions:
contents: 'read'
id-token: 'write'
steps:
- uses: actions/checkout@v3.5.2
with:
persist-credentials: false
fetch-depth: 0
name: Check if cached state disks exist for ${{ inputs.network || vars.ZCASH_NETWORK }}
uses: ./.github/workflows/find-cached-disks.yml
with:
network: ${{ inputs.network || vars.ZCASH_NETWORK }}
# Setup gcloud CLI
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v1.1.0
with:
retries: '3'
workload_identity_provider: '${{ vars.GCP_WIF }}'
service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}'
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v1.1.0
# Disk images in GCP are required to be in lowercase, but the blockchain network
# uses sentence case, so we need to downcase ${{ vars.ZCASH_NETWORK or github.event.inputs.network }}
#
# Passes a lowercase Network name to subsequent steps using $NETWORK env variable
- name: Downcase network name for disks
run: |
NETWORK_CAPS=${{ github.event.inputs.network || vars.ZCASH_NETWORK }}
echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV
# Find a cached state disk for subsequent jobs needing a cached state without
# restricting the result from any branch.
#
# This search is executed considering the actual version from constants.rs
#
# Generate one of the following outputs with a boolean to pass to subsequent jobs:
# - lwd_tip_disk
# - zebra_tip_disk
# - zebra_checkpoint_disk
- name: Find cached state disks
id: get-available-disks
run: |
LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1)
echo "STATE_VERSION: $LOCAL_STATE_VERSION"
LWD_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~lwd-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
if [[ -z "$LWD_TIP_DISK" ]]; then
echo "No TIP disk found for LWD"
echo "lwd_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT"
else
echo "Disk: $LWD_TIP_DISK"
echo "lwd_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT"
fi
ZEBRA_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
if [[ -z "$ZEBRA_TIP_DISK" ]]; then
echo "No TIP disk found for ZEBRA"
echo "zebra_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT"
else
echo "Disk: $ZEBRA_TIP_DISK"
echo "zebra_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT"
fi
ZEBRA_CHECKPOINT_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
if [[ -z "$ZEBRA_CHECKPOINT_DISK" ]]; then
echo "No CHECKPOINT found for ZEBRA"
echo "zebra_checkpoint_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT"
else
echo "Disk: $ZEBRA_CHECKPOINT_DISK"
echo "zebra_checkpoint_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT"
fi
# Check if the cached state disks used by the tests are available for Testnet.
#
# The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml
# Some outputs are ignored, because we don't run those jobs on Testnet.
get-available-disks-testnet:
name: Check if cached state disks exist for Testnet
uses: ./.github/workflows/find-cached-disks.yml
with:
network: 'Testnet'
# Build the docker image used by the tests.
#
# The default network in the Zebra config in the image is Mainnet, unless a manually triggered
# workflow or repository variable is configured differently. Testnet jobs change that config to
# testnet when running the image.
build:
name: Build CI Docker
uses: ./.github/workflows/build-docker-image.yml
@ -163,8 +114,7 @@ jobs:
dockerfile_path: ./docker/Dockerfile
dockerfile_target: tests
image_name: ${{ vars.CI_IMAGE_NAME }}
# The default network in the image is based on the default network variable in GitHub
network: ${{ vars.ZCASH_NETWORK }}
network: ${{ inputs.network || vars.ZCASH_NETWORK }}
checkpoint_sync: true
rust_backtrace: full
rust_lib_backtrace: full

99
.github/workflows/find-cached-disks.yml vendored Normal file
View File

@ -0,0 +1,99 @@
name: Check if cached state disks exist
on:
workflow_call:
inputs:
network:
description: 'The Zcash network used to look up the disks'
required: true
type: string
outputs:
lwd_tip_disk:
description: 'true if there is a lightwalletd and Zebra cached state disk, synced near the chain tip'
value: ${{ jobs.get-available-disks.outputs.lwd_tip_disk }}
zebra_tip_disk:
description: 'true if there is a Zebra cached state disk synced near the chain tip'
value: ${{ jobs.get-available-disks.outputs.zebra_tip_disk }}
zebra_checkpoint_disk:
description: 'true if there is a Zebra cached state disk synced to the mandatory Zebra checkpoint'
value: ${{ jobs.get-available-disks.outputs.zebra_checkpoint_disk }}
jobs:
get-available-disks:
name: Check if cached state disks exist
runs-on: ubuntu-latest
outputs:
lwd_tip_disk: ${{ steps.get-available-disks.outputs.lwd_tip_disk }}
zebra_tip_disk: ${{ steps.get-available-disks.outputs.zebra_tip_disk }}
zebra_checkpoint_disk: ${{ steps.get-available-disks.outputs.zebra_checkpoint_disk }}
permissions:
contents: 'read'
id-token: 'write'
steps:
- uses: actions/checkout@v3.5.2
with:
persist-credentials: false
fetch-depth: 0
# Setup gcloud CLI
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v1.1.0
with:
retries: '3'
workload_identity_provider: '${{ vars.GCP_WIF }}'
service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}'
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v1.1.0
# Disk images in GCP are required to be in lowercase, but the blockchain network
# uses sentence case, so we need to downcase ${{ inputs.network }}
#
# Passes a lowercase Network name to subsequent steps using $NETWORK env variable
- name: Downcase network name for disks
run: |
NETWORK_CAPS=${{ inputs.network }}
echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV
# Check if there are cached state disks available for subsequent jobs to use.
#
# This lookup uses the state version from constants.rs.
# It accepts disks generated by any branch, including draft and unmerged PRs.
#
# If the disk exists, sets the corresponding output to "true":
# - lwd_tip_disk
# - zebra_tip_disk
# - zebra_checkpoint_disk
- name: Check if cached state disks exist
id: get-available-disks
run: |
LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1)
echo "STATE_VERSION: $LOCAL_STATE_VERSION"
LWD_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~lwd-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
if [[ -z "$LWD_TIP_DISK" ]]; then
echo "No TIP disk found for lightwalletd on network: ${NETWORK}"
echo "lwd_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT"
else
echo "Disk: $LWD_TIP_DISK"
echo "lwd_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT"
fi
ZEBRA_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
if [[ -z "$ZEBRA_TIP_DISK" ]]; then
echo "No TIP disk found for Zebra on network: ${NETWORK}"
echo "zebra_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT"
else
echo "Disk: $ZEBRA_TIP_DISK"
echo "zebra_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT"
fi
ZEBRA_CHECKPOINT_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
if [[ -z "$ZEBRA_CHECKPOINT_DISK" ]]; then
echo "No CHECKPOINT disk found for Zebra on network: ${NETWORK}"
echo "zebra_checkpoint_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT"
else
echo "Disk: $ZEBRA_CHECKPOINT_DISK"
echo "zebra_checkpoint_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT"
fi