From 5fa40216df28de2b24c0dd84b1ee49e8dd4e2ea5 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 26 Jan 2022 21:46:18 -0400 Subject: [PATCH] refactor (actions): make better use of variables, secrets and versions (#3393) * style: use global variables and don't double print Remove repeated instances of global environment variables. Do not print ENV variables on the terminal as GitHub Actions already shows it. * fix (actions): Use fixed major versions for actions As actions get recurrent fixes, using a specific version causes more maintance on the pipelines. On the other hand, using @master versions could make some action unreliable, as breaking changes might be included without further notice, and even change behavior on a daily basis. * refactor: make better use of ENV variables A whole step with refex was being used to extract different variables from GitHub's environment. This gets depecrated in favor of using `rlespinasse/github-slug-action@v4` which has slug URL variables. A SLUG on a variable will: - put the variable content in lower case - replace any character by - except 0-9, a-z, ., and _ - remove leading and trailing - character - limit the string size to 63 characters This changes also takes care of using the Head or Base branch for deployments. This will allow us tomerge of workflows, as most steps on this deployment actions are very similar, with little variations between workflows. * fix (actions): use secrets for sensitive information * revert: use specific versions for dependabot Reverting commit 8c934099028e0651e464678c096d8d3815efe95c --- .github/workflows/cd.yml | 78 ++++++------------- .github/workflows/ci.yml | 60 +------------- .github/workflows/coverage.yml | 20 +---- .github/workflows/manual-deploy.yml | 44 ++++------- .../regenerate-stateful-test-disks.yml | 51 +++++------- .github/workflows/test.yml | 48 ++++-------- .github/workflows/zcashd-manual-deploy.yml | 37 ++++----- 7 files changed, 99 insertions(+), 239 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 0c6ee71d4..0ed6421bf 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -7,13 +7,13 @@ on: - main env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} REGION: us-east1 ZONE: us-east1-b - + MACHINE_TYPE: n2d-standard-4 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: - build: name: Build images timeout-minutes: 60 @@ -23,20 +23,13 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 # Setup gcloud CLI - name: Set up gcloud SDK environment - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} @@ -46,7 +39,7 @@ jobs: run: | gcloud builds submit \ --config cloudbuild.yaml \ - --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME" + --substitutions SHORT_SHA="${{ env.GITHUB_SHA_SHORT }}",BRANCH_NAME="${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" deploy: name: Deploy mainnet nodes @@ -58,70 +51,45 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 # Setup gcloud CLI - name: Set up gcloud SDK environment - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} - # Run once: create firewall rule to allow incoming traffic to the nodes - # - name: Create Zcash incoming traffic firewall rule - # run: | - # gcloud compute firewall-rules create "allow-zcash" \ - # --target-tags zebrad \ - # --allow tcp:8233,tcp:18233 \ - # --source-ranges 0.0.0.0/0 \ - # --description="Allow incoming Zcash traffic from anywhere" \ - # Create instance template from container image - name: Create instance template run: | - gcloud compute instance-templates create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \ - --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \ - --create-disk name=zebrad-cache-$SHORT_SHA,auto-delete=yes,size=100GB,type=pd-balanced \ - --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-$SHORT_SHA \ - --machine-type n2d-standard-4 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + gcloud compute instance-templates create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --container-image "gcr.io/${{ env.PROJECT_ID}}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}" \ + --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-balanced \ + --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ --tags zebrad \ - # Run once: create firewall rule to allow healthchecks - # - name: Create healthcheck firewall rule - # run: | - # gcloud compute firewall-rules create "allow-tracing-health-check" \ - # --target-tags zebrad \ - # --allow tcp:3000 \ - # --source-ranges 130.211.0.0/22,35.191.0.0/16 \ - # --description="Allow HTTP requests to our tracing endpoint from Google's probes" \ - # Check if our destination instance group exists already - name: Check if instance group exists id: does-group-exist continue-on-error: true run: | - gcloud compute instance-groups list | grep "zebrad-$BRANCH_NAME" | grep "$REGION" + gcloud compute instance-groups list | grep "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" | grep "${{ env.REGION }}" # Deploy new managed instance group using the new instance template - name: Create managed instance group if: steps.does-group-exist.outcome == 'failure' run: | gcloud compute instance-groups managed create \ - "zebrad-$BRANCH_NAME" \ - --template "zebrad-$BRANCH_NAME-$SHORT_SHA" \ + "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" \ + --template "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ --health-check zebrad-tracing-filter \ --initial-delay 30 \ - --region "$REGION" \ + --region "${{ env.REGION }}" \ --size 2 # Rolls out update to existing group using the new instance template @@ -129,6 +97,6 @@ jobs: if: steps.does-group-exist.outcome == 'success' run: | gcloud compute instance-groups managed rolling-action start-update \ - "zebrad-$BRANCH_NAME" \ - --version template="zebrad-$BRANCH_NAME-$SHORT_SHA" \ - --region "$REGION" + "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" \ + --version template="zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --region "${{ env.REGION }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10746bcc6..7a623d86a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,10 @@ on: - 'docker/**' - '.github/workflows/ci.yml' +env: + CARGO_INCREMENTAL: 0 + RUST_BACKTRACE: full + jobs: test: name: Test (+${{ matrix.rust }}) on ${{ matrix.os }} @@ -25,9 +29,6 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] rust: [stable] - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full steps: - uses: actions/checkout@v2.4.0 @@ -99,19 +100,6 @@ jobs: working-directory: ./zebra-consensus run: cargo run --example download-params - - name: Show env vars - run: | - echo "Test env vars:" - echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" - echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}" - echo "PROPTEST_CASES=${{ env.PROPTEST_CASES }}" - echo "PROPTEST_MAX_SHRINK_ITERS=${{ env.PROPTEST_MAX_SHRINK_ITERS }}" - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - - name: Run tests uses: actions-rs/cargo@v1.0.3 with: @@ -136,9 +124,6 @@ jobs: strategy: matrix: rust: [stable] - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full steps: - uses: actions/checkout@v2.4.0 @@ -158,19 +143,6 @@ jobs: with: command: fetch - - name: Show env vars - run: | - echo "Test env vars:" - echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" - echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}" - echo "PROPTEST_CASES=${{ env.PROPTEST_CASES }}" - echo "PROPTEST_MAX_SHRINK_ITERS=${{ env.PROPTEST_MAX_SHRINK_ITERS }}" - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - # This test changes zebra-chain's activation heights, # which can recompile all the Zebra crates, # so we want its build products to be cached separately. @@ -193,10 +165,6 @@ jobs: strategy: matrix: rust: [stable, beta] - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full - steps: - uses: actions/checkout@v2.4.0 @@ -216,14 +184,6 @@ jobs: with: command: fetch - - name: Show env vars - run: | - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - - name: Run build without features enabled working-directory: ./zebra-chain run: cargo build --verbose --no-default-features @@ -233,10 +193,6 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-latest - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full - steps: - uses: actions/checkout@v2.4.0 with: @@ -262,14 +218,6 @@ jobs: with: command: fetch - - name: Show env vars - run: | - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - - name: Build uses: actions-rs/cargo@v1.0.3 with: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 086c06073..cb83b5583 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,6 +11,10 @@ on: - 'codecov.yml' - '.github/workflows/coverage.yml' +env: + CARGO_INCREMENTAL: 0 + RUST_BACKTRACE: full + jobs: coverage: name: Coverage (+nightly) @@ -19,9 +23,6 @@ jobs: # - parameter downloads (40 minutes, but only when the cache expires) timeout-minutes: 115 runs-on: ubuntu-latest - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full steps: - uses: actions/checkout@v2.4.0 @@ -74,19 +75,6 @@ jobs: working-directory: ./zebra-consensus run: cargo llvm-cov --lcov --no-report run --example download-params - - name: Show env vars - run: | - echo "Test env vars:" - echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" - echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}" - echo "PROPTEST_CASES=${{ env.PROPTEST_CASES }}" - echo "PROPTEST_MAX_SHRINK_ITERS=${{ env.PROPTEST_MAX_SHRINK_ITERS }}" - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - - name: Run Zebra tests run: cargo llvm-cov --lcov --no-report diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 93a35dae2..47087393c 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -9,10 +9,12 @@ on: default: true env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + ZONE: us-central1-a + MACHINE_TYPE: n2-standard-4 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: - deploy: name: Deploy one zebrad node runs-on: ubuntu-latest @@ -21,19 +23,12 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 - name: Set up gcloud - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} @@ -43,25 +38,16 @@ jobs: run: | gcloud builds submit \ --config cloudbuild.yaml \ - --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME",_CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",_NETWORK="${{ github.event.inputs.network }}" - - # Run once: create firewall rule to allow incoming traffic to the node - # - name: Create Zcash incoming traffic firewall rule - # run: | - # gcloud compute firewall-rules create "allow-zcash" \ - # --target-tags zebrad \ - # --allow tcp:8233,tcp:18233 \ - # --source-ranges 0.0.0.0/0 \ - # --description="Allow incoming Zcash traffic from anywhere" \ + --substitutions SHORT_SHA="${{ env.GITHUB_SHA_SHORT }}",BRANCH_NAME="${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}",_CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",_NETWORK="${{ github.event.inputs.network }}" # Creates Compute Engine virtual machine instance w/ zebrad container and disks - name: Create instance running zebrad container image run: | - gcloud compute instances create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \ - --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \ - --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \ - --create-disk name=zebrad-cache-$SHORT_SHA,auto-delete=yes,size=100GB,type=pd-balanced \ - --machine-type n2-standard-4 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --container-image "gcr.io/${{ env.PROJECT_ID }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}" \ + --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \ + --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-balanced \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --tags zebrad \ - --zone us-central1-a + --zone ${{ env.ZONE }} diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml index 388b1ffbe..e81bcdaae 100644 --- a/.github/workflows/regenerate-stateful-test-disks.yml +++ b/.github/workflows/regenerate-stateful-test-disks.yml @@ -7,8 +7,10 @@ on: default: 'mainnet' env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} ZONE: europe-west1-b + MACHINE_TYPE: n2-standard-4 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: @@ -20,64 +22,49 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + - name: Set up gcloud - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} - # Run once: create firewall rule to allow incoming traffic to the nodes - # - name: Create Zcash incoming traffic firewall rule - # run: | - # gcloud compute firewall-rules create "allow-zcash" \ - # --target-tags zebrad \ - # --allow tcp:8233,tcp:18233 \ - # --source-ranges 0.0.0.0/0 \ - # --description="Allow incoming Zcash traffic from anywhere" \ - # Creates Compute Engine virtual machine instance w/ disks - name: Create instance run: | - gcloud compute instances create-with-container "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" \ + gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 100GB \ --boot-disk-type pd-ssd \ --container-image rust:buster \ - --container-mount-disk mount-path='/${{ github.event.inputs.network }}',name="zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy" \ + --container-mount-disk mount-path='/${{ github.event.inputs.network }}',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" \ --container-restart-policy never \ - --create-disk name="zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-balanced \ - --machine-type n2-standard-4 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-balanced \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ --tags zebrad \ - --zone "$ZONE" + --zone "${{ env.ZONE }}" # Build and run test container to sync up to activation and no further - name: Regenerate state for tests id: regenerate-state run: | - gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" --command \ - "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git && + gcloud compute ssh "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --zone "${{ env.ZONE }}" --command \ + "git clone -b ${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} https://github.com/ZcashFoundation/zebra.git && cd zebra/ && - docker build --build-arg SHORT_SHA=$SHORT_SHA -f docker/Dockerfile.test -t zebrad-test . && - docker run -i -e "ZEBRA_SKIP_IPV6_TESTS=1" --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network }}; + docker build --build-arg SHORT_SHA=${{ env.GITHUB_SHA_SHORT }} -f docker/Dockerfile.test -t zebrad-test . && + docker run -i -e "ZEBRA_SKIP_IPV6_TESTS=1" --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network }}; " # Create image from disk that will be used in test.yml workflow - name: Create image from state disk # Only run if the earlier step succeeds if: steps.regenerate-state.outcome == 'success' run: | - gcloud compute images create "zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy" --source-disk="zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy" --source-disk-zone="$ZONE" + gcloud compute images create "zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" --source-disk="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" --source-disk-zone="${{ env.ZONE }}" # Clean up - name: Delete test instance # Always run even if the earlier step fails if: ${{ always() }} run: | - gcloud compute instances delete "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone "$ZONE" + gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a7df67cf..39dca7992 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,8 +14,10 @@ on: - '.github/workflows/test.yml' env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} ZONE: europe-west1-b + MACHINE_TYPE: n2-standard-8 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: @@ -27,60 +29,44 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 - name: Set up gcloud - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} - # Run once: create firewall rule to allow incoming traffic to the nodes - # - name: Create Zcash incoming traffic firewall rule - # run: | - # gcloud compute firewall-rules create "allow-zcash" \ - # --target-tags zebrad \ - # --allow tcp:8233,tcp:18233 \ - # --source-ranges 0.0.0.0/0 \ - # --description="Allow incoming Zcash traffic from anywhere" \ - # Creates Compute Engine virtual machine instance w/ disks - name: Create instance run: | - gcloud compute instances create-with-container "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" \ + gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 100GB \ --boot-disk-type pd-ssd \ --container-image rust:buster \ - --container-mount-disk mount-path='/mainnet',name="zebrad-cache-$SHORT_SHA-mainnet-canopy" \ + --container-mount-disk mount-path='/mainnet',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy" \ --container-restart-policy never \ - --create-disk name="zebrad-cache-$SHORT_SHA-mainnet-canopy",image=zebrad-cache-1558f3378-mainnet-canopy \ - --machine-type n2-standard-8 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy",image=zebrad-cache-1558f3378-mainnet-canopy \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ --tags zebrad \ - --zone "$ZONE" + --zone "${{ env.ZONE }}" # Build and run test container - name: Run all tests run: | - gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --ssh-flag="-o ServerAliveInterval=5" --zone "$ZONE" --command \ - "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git && + gcloud compute ssh "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" --command \ + "git clone -b ${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} https://github.com/ZcashFoundation/zebra.git && cd zebra/ && - docker build --build-arg SHORT_SHA=$SHORT_SHA -f docker/Dockerfile.test -t zebrad-test . && + docker build --build-arg SHORT_SHA=${{ env.GITHUB_SHA_SHORT }} -f docker/Dockerfile.test -t zebrad-test . && docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 zebrad-test:latest cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored && - docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-mainnet-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet + docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet " # Clean up - name: Delete test instance # Always run even if the earlier step fails if: ${{ always() }} run: | - gcloud compute instances delete "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone "$ZONE" + gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}" diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml index e56fc89e7..73e1973c9 100644 --- a/.github/workflows/zcashd-manual-deploy.yml +++ b/.github/workflows/zcashd-manual-deploy.yml @@ -9,8 +9,10 @@ on: default: 10 env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} REGION: us-central1 + MACHINE_TYPE: n2-standard-4 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: @@ -22,31 +24,26 @@ jobs: - uses: actions/checkout@v2.4.0 with: persist-credentials: false - - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 # Setup gcloud CLI - name: Set up gcloud SDK environment - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} # Create instance template from container image - name: Create instance template run: | - gcloud compute instance-templates create-with-container "zcashd-$BRANCH_NAME-$SHORT_SHA" \ + gcloud compute instance-templates create-with-container "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 100GB \ --container-image "electriccoinco/zcashd" \ --container-env ZCASHD_NETWORK="${{ github.event.inputs.network }}" \ - --machine-type n2-standard-4 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ --tags zcashd \ @@ -55,16 +52,16 @@ jobs: id: does-group-exist continue-on-error: true run: | - gcloud compute instance-groups list | grep "zcashd-$BRANCH_NAME-${{ github.event.inputs.network }}" | grep "$REGION" + gcloud compute instance-groups list | grep "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ github.event.inputs.network }}" | grep "${{ env.REGION }}" # Deploy new managed instance group using the new instance template - name: Create managed instance group if: steps.does-group-exist.outcome == 'failure' run: | gcloud compute instance-groups managed create \ - "zcashd-$BRANCH_NAME-${{ github.event.inputs.network }}" \ - --template "zcashd-$BRANCH_NAME-$SHORT_SHA" \ - --region "$REGION" \ + "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ github.event.inputs.network }}" \ + --template "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --region "${{ env.REGION }}" \ --size "${{ github.event.inputs.size }}" # Rolls out update to existing group using the new instance template @@ -72,6 +69,6 @@ jobs: if: steps.does-group-exist.outcome == 'success' run: | gcloud compute instance-groups managed rolling-action start-update \ - "zcashd-$BRANCH_NAME-${{ github.event.inputs.network }}" \ - --version template="zcashd-$BRANCH_NAME-$SHORT_SHA" \ - --region "$REGION" + "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ github.event.inputs.network }}" \ + --version template="zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --region "${{ env.REGION }}"