From a763eec9f3915818028b9db0daaef83cffff2e73 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 26 Nov 2022 07:11:22 +1000 Subject: [PATCH] fix(ci): Fix network parameter in continous-delivery.yml, and add network labels to GCP jobs (#5710) * chore: add Network as a label * Fix network parameter in continous-delivery.yml * Standardise network usage in zcashd-manual-deploy * Use lowercase network labels * Fix some shellcheck errors * Hard-code a Mainnet default to support contexts where env is not available * Fix string syntax * Fix more shellcheck errors * Update .github/workflows/zcashd-manual-deploy.yml Co-authored-by: Gustavo Valverde Co-authored-by: Arya --- .github/workflows/continous-delivery.yml | 23 +++++++++--- .github/workflows/deploy-gcp-tests.yml | 43 ++++++++++++++-------- .github/workflows/zcashd-manual-deploy.yml | 21 ++++++++--- 3 files changed, 60 insertions(+), 27 deletions(-) diff --git a/.github/workflows/continous-delivery.yml b/.github/workflows/continous-delivery.yml index 4b6bf77cb..04828142e 100644 --- a/.github/workflows/continous-delivery.yml +++ b/.github/workflows/continous-delivery.yml @@ -26,7 +26,6 @@ on: - published env: - NETWORK: Mainnet GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra REGION: us-central1 ZONE: us-central1-a @@ -40,7 +39,7 @@ jobs: # - If our semver is `v1.3.0` the resulting output from this job would be `v1` # # Note: We just use the first part of the version to replace old instances, and change - # it when a major version is released, to keep a segregation between new and old + # it when a major version is released, to keep a segregation between new and old # versions. versioning: name: Versioning @@ -71,7 +70,8 @@ jobs: dockerfile_path: ./docker/Dockerfile dockerfile_target: runtime image_name: zebrad - network: Mainnet + # We need to hard-code Mainnet here, because env is not allowed in this context + network: ${{ inputs.network || 'Mainnet' }} checkpoint_sync: true rust_backtrace: '1' zebra_skip_ipv6_tests: '1' @@ -89,7 +89,7 @@ jobs: # - on every push/merge to the `main` branch # - on every release, when it's published deploy-nodes: - name: Deploy Mainnet nodes + name: Deploy ${{ inputs.network || 'Mainnet' }} nodes needs: [ build, versioning ] runs-on: ubuntu-latest timeout-minutes: 30 @@ -104,6 +104,17 @@ jobs: with: short-length: 7 + # Makes the Zcash network name lowercase. + # + # Labels in GCP are required to be in lowercase, but the blockchain network + # uses sentence case, so we need to downcase ${{ inputs.network || 'Mainnet' }}. + # + # Passes the lowercase network to subsequent steps using $NETWORK env variable. + - name: Downcase network name for labels + run: | + NETWORK_CAPS="${{ inputs.network || 'Mainnet' }}" + echo "NETWORK=${NETWORK_CAPS,,}" >> "$GITHUB_ENV" + # Setup gcloud CLI - name: Authenticate to Google Cloud id: auth @@ -129,7 +140,7 @@ jobs: --container-mount-disk=mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \ --machine-type ${{ env.MACHINE_TYPE }} \ --scopes cloud-platform \ - --labels=app=zebrad,environment=prod,github_ref=${{ env.GITHUB_REF_SLUG_URL }} \ + --labels=app=zebrad,environment=prod,network=${NETWORK},github_ref=${{ env.GITHUB_REF_SLUG_URL }} \ --tags zebrad # Check if our destination instance group exists already @@ -210,5 +221,5 @@ jobs: --container-mount-disk=mount-path='/zebrad-cache' \ --machine-type ${{ env.MACHINE_TYPE }} \ --zone ${{ env.ZONE }} \ - --labels=app=zebrad,environment=qa,github_ref=${{ env.GITHUB_REF_SLUG_URL }} \ + --labels=app=zebrad,environment=qa,network=${NETWORK},github_ref=${{ env.GITHUB_REF_SLUG_URL }} \ --tags zebrad diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index 2152633e2..6782a3357 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -125,6 +125,17 @@ jobs: with: short-length: 7 + # Makes the Zcash network name lowercase. + # + # Labels in GCP are required to be in lowercase, but the blockchain network + # uses sentence case, so we need to downcase ${{ inputs.network }}. + # + # Passes ${{ inputs.network }} to subsequent steps using $NETWORK env variable. + - name: Downcase network name for labels + run: | + NETWORK_CAPS="${{ inputs.network }}" + echo "NETWORK=${NETWORK_CAPS,,}" >> "$GITHUB_ENV" + # Install our SSH secret - name: Install private SSH key uses: shimataro/ssh-key-action@v2.4.0 @@ -163,7 +174,7 @@ jobs: --scopes cloud-platform \ --metadata=google-monitoring-enabled=TRUE,google-logging-enabled=TRUE \ --metadata-from-file=startup-script=.github/workflows/scripts/gcp-vm-startup-script.sh \ - --labels=app=${{ inputs.app_name }},environment=test,github_ref=${{ env.GITHUB_REF_SLUG_URL }},test=${{ inputs.test_id }} \ + --labels=app=${{ inputs.app_name }},environment=test,network=${NETWORK},github_ref=${{ env.GITHUB_REF_SLUG_URL }},test=${{ inputs.test_id }} \ --tags ${{ inputs.app_name }} \ --zone ${{ env.ZONE }} sleep 60 @@ -272,10 +283,10 @@ jobs: with: short-length: 7 - - name: Downcase network name for disks + - name: Downcase network name for disks and labels run: | - NETWORK_CAPS=${{ inputs.network }} - echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + NETWORK_CAPS="${{ inputs.network }}" + echo "NETWORK=${NETWORK_CAPS,,}" >> "$GITHUB_ENV" # Install our SSH secret - name: Install private SSH key @@ -385,8 +396,8 @@ jobs: echo "Selected Disk: $CACHED_DISK_NAME" echo "::set-output name=cached_disk_name::$CACHED_DISK_NAME" - echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV - echo "CACHED_DISK_NAME=$CACHED_DISK_NAME" >> $GITHUB_ENV + echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> "$GITHUB_ENV" + echo "CACHED_DISK_NAME=$CACHED_DISK_NAME" >> "$GITHUB_ENV" # Create a Compute Engine virtual machine and attach a cached state disk using the # $CACHED_DISK_NAME variable as the source image to populate the disk cached state @@ -404,7 +415,7 @@ jobs: --scopes cloud-platform \ --metadata=google-monitoring-enabled=TRUE,google-logging-enabled=TRUE \ --metadata-from-file=startup-script=.github/workflows/scripts/gcp-vm-startup-script.sh \ - --labels=app=${{ inputs.app_name }},environment=test,github_ref=${{ env.GITHUB_REF_SLUG_URL }},test=${{ inputs.test_id }} \ + --labels=app=${{ inputs.app_name }},environment=test,network=${NETWORK},github_ref=${{ env.GITHUB_REF_SLUG_URL }},test=${{ inputs.test_id }} \ --tags ${{ inputs.app_name }} \ --zone ${{ env.ZONE }} sleep 60 @@ -1300,10 +1311,10 @@ jobs: # Passes ${{ env.GITHUB_REF_SLUG_URL }} to subsequent steps using $SHORT_GITHUB_REF env variable. - name: Format network name and branch name for disks run: | - NETWORK_CAPS=${{ inputs.network }} - echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV - LONG_GITHUB_REF=${{ env.GITHUB_REF_SLUG_URL }} - echo "SHORT_GITHUB_REF=${LONG_GITHUB_REF:0:12}" >> $GITHUB_ENV + NETWORK_CAPS="${{ inputs.network }}" + echo "NETWORK=${NETWORK_CAPS,,}" >> "$GITHUB_ENV" + LONG_GITHUB_REF="${{ env.GITHUB_REF_SLUG_URL }}" + echo "SHORT_GITHUB_REF=${LONG_GITHUB_REF:0:12}" >> "$GITHUB_ENV" # Install our SSH secret - name: Install private SSH key @@ -1336,7 +1347,7 @@ jobs: LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" $GITHUB_WORKSPACE/zebra-state/src/constants.rs | grep -oE "[0-9]+" | tail -n1) echo "STATE_VERSION: $LOCAL_STATE_VERSION" - echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV + echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> "$GITHUB_ENV" # Sets the $UPDATE_SUFFIX env var to "-u" if updating a previous cached state, # and the empty string otherwise. @@ -1358,8 +1369,8 @@ jobs: # We're going to delete old images after a few days, so we only need the time here TIME_SUFFIX=$(date '+%H%M%S' --utc) - echo "UPDATE_SUFFIX=$UPDATE_SUFFIX" >> $GITHUB_ENV - echo "TIME_SUFFIX=$TIME_SUFFIX" >> $GITHUB_ENV + echo "UPDATE_SUFFIX=$UPDATE_SUFFIX" >> "$GITHUB_ENV" + echo "TIME_SUFFIX=$TIME_SUFFIX" >> "$GITHUB_ENV" # Get the sync height from the test logs, which is later used as part of the # disk description and labels. @@ -1401,7 +1412,7 @@ jobs: fi echo "Found sync height in logs: $SYNC_HEIGHT" - echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV + echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> "$GITHUB_ENV" # Get the original cached state height from google cloud. # @@ -1422,7 +1433,7 @@ jobs: echo "$CACHED_DISK_NAME height: $ORIGINAL_HEIGHT" fi - echo "ORIGINAL_HEIGHT=$ORIGINAL_HEIGHT" >> $GITHUB_ENV + echo "ORIGINAL_HEIGHT=$ORIGINAL_HEIGHT" >> "$GITHUB_ENV" # Create an image from the state disk, which will be used for any tests that start # after it is created. These tests can be in the same workflow, or in a different PR. diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml index e4e2dfdd0..6c19e04e3 100644 --- a/.github/workflows/zcashd-manual-deploy.yml +++ b/.github/workflows/zcashd-manual-deploy.yml @@ -37,6 +37,17 @@ jobs: with: short-length: 7 + # Makes the Zcash network name lowercase. + # + # Labels in GCP are required to be in lowercase, but the blockchain network + # uses sentence case, so we need to downcase ${{ inputs.network }}. + # + # Passes ${{ inputs.network }} to subsequent steps using $NETWORK env variable. + - name: Downcase network name for labels + run: | + NETWORK_CAPS="${{ inputs.network }}" + echo "NETWORK=${NETWORK_CAPS,,}" >> "$GITHUB_ENV" + # Setup gcloud CLI - name: Authenticate to Google Cloud id: auth @@ -60,11 +71,11 @@ jobs: --container-stdin \ --container-tty \ --container-image electriccoinco/zcashd \ - --container-env ZCASHD_NETWORK="${{ github.event.inputs.network }}" \ + --container-env ZCASHD_NETWORK="${{ inputs.network }}" \ --machine-type ${{ env.MACHINE_TYPE }} \ --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ - --labels=app=zcashd,environment=prod,github_ref=${{ env.GITHUB_REF_SLUG_URL }} \ + --labels=app=zcashd,environment=prod,network=${NETWORK},github_ref=${{ env.GITHUB_REF_SLUG_URL }} \ --tags zcashd # Check if our destination instance group exists already @@ -72,14 +83,14 @@ jobs: id: does-group-exist continue-on-error: true run: | - gcloud compute instance-groups list | grep "zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ github.event.inputs.network }}" | grep "${{ env.REGION }}" + gcloud compute instance-groups list | grep "zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ inputs.network }}" | grep "${{ env.REGION }}" # Deploy new managed instance group using the new instance template - name: Create managed instance group if: steps.does-group-exist.outcome == 'failure' run: | gcloud compute instance-groups managed create \ - "zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ github.event.inputs.network }}" \ + "zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ inputs.network }}" \ --template "zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ --region "${{ env.REGION }}" \ --size "${{ github.event.inputs.size }}" @@ -89,6 +100,6 @@ jobs: if: steps.does-group-exist.outcome == 'success' run: | gcloud compute instance-groups managed rolling-action start-update \ - "zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ github.event.inputs.network }}" \ + "zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ inputs.network }}" \ --version template="zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ --region "${{ env.REGION }}"