diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 0c6ee71d4..0ed6421bf 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -7,13 +7,13 @@ on: - main env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} REGION: us-east1 ZONE: us-east1-b - + MACHINE_TYPE: n2d-standard-4 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: - build: name: Build images timeout-minutes: 60 @@ -23,20 +23,13 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 # Setup gcloud CLI - name: Set up gcloud SDK environment - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} @@ -46,7 +39,7 @@ jobs: run: | gcloud builds submit \ --config cloudbuild.yaml \ - --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME" + --substitutions SHORT_SHA="${{ env.GITHUB_SHA_SHORT }}",BRANCH_NAME="${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" deploy: name: Deploy mainnet nodes @@ -58,70 +51,45 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 # Setup gcloud CLI - name: Set up gcloud SDK environment - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} - # Run once: create firewall rule to allow incoming traffic to the nodes - # - name: Create Zcash incoming traffic firewall rule - # run: | - # gcloud compute firewall-rules create "allow-zcash" \ - # --target-tags zebrad \ - # --allow tcp:8233,tcp:18233 \ - # --source-ranges 0.0.0.0/0 \ - # --description="Allow incoming Zcash traffic from anywhere" \ - # Create instance template from container image - name: Create instance template run: | - gcloud compute instance-templates create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \ - --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \ - --create-disk name=zebrad-cache-$SHORT_SHA,auto-delete=yes,size=100GB,type=pd-balanced \ - --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-$SHORT_SHA \ - --machine-type n2d-standard-4 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + gcloud compute instance-templates create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --container-image "gcr.io/${{ env.PROJECT_ID}}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}" \ + --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-balanced \ + --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ --tags zebrad \ - # Run once: create firewall rule to allow healthchecks - # - name: Create healthcheck firewall rule - # run: | - # gcloud compute firewall-rules create "allow-tracing-health-check" \ - # --target-tags zebrad \ - # --allow tcp:3000 \ - # --source-ranges 130.211.0.0/22,35.191.0.0/16 \ - # --description="Allow HTTP requests to our tracing endpoint from Google's probes" \ - # Check if our destination instance group exists already - name: Check if instance group exists id: does-group-exist continue-on-error: true run: | - gcloud compute instance-groups list | grep "zebrad-$BRANCH_NAME" | grep "$REGION" + gcloud compute instance-groups list | grep "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" | grep "${{ env.REGION }}" # Deploy new managed instance group using the new instance template - name: Create managed instance group if: steps.does-group-exist.outcome == 'failure' run: | gcloud compute instance-groups managed create \ - "zebrad-$BRANCH_NAME" \ - --template "zebrad-$BRANCH_NAME-$SHORT_SHA" \ + "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" \ + --template "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ --health-check zebrad-tracing-filter \ --initial-delay 30 \ - --region "$REGION" \ + --region "${{ env.REGION }}" \ --size 2 # Rolls out update to existing group using the new instance template @@ -129,6 +97,6 @@ jobs: if: steps.does-group-exist.outcome == 'success' run: | gcloud compute instance-groups managed rolling-action start-update \ - "zebrad-$BRANCH_NAME" \ - --version template="zebrad-$BRANCH_NAME-$SHORT_SHA" \ - --region "$REGION" + "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" \ + --version template="zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --region "${{ env.REGION }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10746bcc6..7a623d86a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,10 @@ on: - 'docker/**' - '.github/workflows/ci.yml' +env: + CARGO_INCREMENTAL: 0 + RUST_BACKTRACE: full + jobs: test: name: Test (+${{ matrix.rust }}) on ${{ matrix.os }} @@ -25,9 +29,6 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] rust: [stable] - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full steps: - uses: actions/checkout@v2.4.0 @@ -99,19 +100,6 @@ jobs: working-directory: ./zebra-consensus run: cargo run --example download-params - - name: Show env vars - run: | - echo "Test env vars:" - echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" - echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}" - echo "PROPTEST_CASES=${{ env.PROPTEST_CASES }}" - echo "PROPTEST_MAX_SHRINK_ITERS=${{ env.PROPTEST_MAX_SHRINK_ITERS }}" - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - - name: Run tests uses: actions-rs/cargo@v1.0.3 with: @@ -136,9 +124,6 @@ jobs: strategy: matrix: rust: [stable] - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full steps: - uses: actions/checkout@v2.4.0 @@ -158,19 +143,6 @@ jobs: with: command: fetch - - name: Show env vars - run: | - echo "Test env vars:" - echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" - echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}" - echo "PROPTEST_CASES=${{ env.PROPTEST_CASES }}" - echo "PROPTEST_MAX_SHRINK_ITERS=${{ env.PROPTEST_MAX_SHRINK_ITERS }}" - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - # This test changes zebra-chain's activation heights, # which can recompile all the Zebra crates, # so we want its build products to be cached separately. @@ -193,10 +165,6 @@ jobs: strategy: matrix: rust: [stable, beta] - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full - steps: - uses: actions/checkout@v2.4.0 @@ -216,14 +184,6 @@ jobs: with: command: fetch - - name: Show env vars - run: | - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - - name: Run build without features enabled working-directory: ./zebra-chain run: cargo build --verbose --no-default-features @@ -233,10 +193,6 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-latest - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full - steps: - uses: actions/checkout@v2.4.0 with: @@ -262,14 +218,6 @@ jobs: with: command: fetch - - name: Show env vars - run: | - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - - name: Build uses: actions-rs/cargo@v1.0.3 with: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 086c06073..cb83b5583 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,6 +11,10 @@ on: - 'codecov.yml' - '.github/workflows/coverage.yml' +env: + CARGO_INCREMENTAL: 0 + RUST_BACKTRACE: full + jobs: coverage: name: Coverage (+nightly) @@ -19,9 +23,6 @@ jobs: # - parameter downloads (40 minutes, but only when the cache expires) timeout-minutes: 115 runs-on: ubuntu-latest - env: - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: full steps: - uses: actions/checkout@v2.4.0 @@ -74,19 +75,6 @@ jobs: working-directory: ./zebra-consensus run: cargo llvm-cov --lcov --no-report run --example download-params - - name: Show env vars - run: | - echo "Test env vars:" - echo "ZEBRA_SKIP_NETWORK_TESTS=${{ env.ZEBRA_SKIP_NETWORK_TESTS }}" - echo "ZCASH_PARAMS=${{ env.ZCASH_PARAMS }}" - echo "PROPTEST_CASES=${{ env.PROPTEST_CASES }}" - echo "PROPTEST_MAX_SHRINK_ITERS=${{ env.PROPTEST_MAX_SHRINK_ITERS }}" - echo "Common env vars:" - echo "RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}" - echo "Build env vars:" - echo "CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}" - echo "CARGO_TARGET_DIR=${{ env.CARGO_TARGET_DIR }}" - - name: Run Zebra tests run: cargo llvm-cov --lcov --no-report diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 93a35dae2..47087393c 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -9,10 +9,12 @@ on: default: true env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + ZONE: us-central1-a + MACHINE_TYPE: n2-standard-4 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: - deploy: name: Deploy one zebrad node runs-on: ubuntu-latest @@ -21,19 +23,12 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 - name: Set up gcloud - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} @@ -43,25 +38,16 @@ jobs: run: | gcloud builds submit \ --config cloudbuild.yaml \ - --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME",_CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",_NETWORK="${{ github.event.inputs.network }}" - - # Run once: create firewall rule to allow incoming traffic to the node - # - name: Create Zcash incoming traffic firewall rule - # run: | - # gcloud compute firewall-rules create "allow-zcash" \ - # --target-tags zebrad \ - # --allow tcp:8233,tcp:18233 \ - # --source-ranges 0.0.0.0/0 \ - # --description="Allow incoming Zcash traffic from anywhere" \ + --substitutions SHORT_SHA="${{ env.GITHUB_SHA_SHORT }}",BRANCH_NAME="${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}",_CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",_NETWORK="${{ github.event.inputs.network }}" # Creates Compute Engine virtual machine instance w/ zebrad container and disks - name: Create instance running zebrad container image run: | - gcloud compute instances create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \ - --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \ - --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \ - --create-disk name=zebrad-cache-$SHORT_SHA,auto-delete=yes,size=100GB,type=pd-balanced \ - --machine-type n2-standard-4 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --container-image "gcr.io/${{ env.PROJECT_ID }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}" \ + --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \ + --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-balanced \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --tags zebrad \ - --zone us-central1-a + --zone ${{ env.ZONE }} diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml index 388b1ffbe..e81bcdaae 100644 --- a/.github/workflows/regenerate-stateful-test-disks.yml +++ b/.github/workflows/regenerate-stateful-test-disks.yml @@ -7,8 +7,10 @@ on: default: 'mainnet' env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} ZONE: europe-west1-b + MACHINE_TYPE: n2-standard-4 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: @@ -20,64 +22,49 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + - name: Set up gcloud - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} - # Run once: create firewall rule to allow incoming traffic to the nodes - # - name: Create Zcash incoming traffic firewall rule - # run: | - # gcloud compute firewall-rules create "allow-zcash" \ - # --target-tags zebrad \ - # --allow tcp:8233,tcp:18233 \ - # --source-ranges 0.0.0.0/0 \ - # --description="Allow incoming Zcash traffic from anywhere" \ - # Creates Compute Engine virtual machine instance w/ disks - name: Create instance run: | - gcloud compute instances create-with-container "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" \ + gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 100GB \ --boot-disk-type pd-ssd \ --container-image rust:buster \ - --container-mount-disk mount-path='/${{ github.event.inputs.network }}',name="zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy" \ + --container-mount-disk mount-path='/${{ github.event.inputs.network }}',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" \ --container-restart-policy never \ - --create-disk name="zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-balanced \ - --machine-type n2-standard-4 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-balanced \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ --tags zebrad \ - --zone "$ZONE" + --zone "${{ env.ZONE }}" # Build and run test container to sync up to activation and no further - name: Regenerate state for tests id: regenerate-state run: | - gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" --command \ - "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git && + gcloud compute ssh "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --zone "${{ env.ZONE }}" --command \ + "git clone -b ${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} https://github.com/ZcashFoundation/zebra.git && cd zebra/ && - docker build --build-arg SHORT_SHA=$SHORT_SHA -f docker/Dockerfile.test -t zebrad-test . && - docker run -i -e "ZEBRA_SKIP_IPV6_TESTS=1" --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network }}; + docker build --build-arg SHORT_SHA=${{ env.GITHUB_SHA_SHORT }} -f docker/Dockerfile.test -t zebrad-test . && + docker run -i -e "ZEBRA_SKIP_IPV6_TESTS=1" --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network }}; " # Create image from disk that will be used in test.yml workflow - name: Create image from state disk # Only run if the earlier step succeeds if: steps.regenerate-state.outcome == 'success' run: | - gcloud compute images create "zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy" --source-disk="zebrad-cache-$SHORT_SHA-${{ github.event.inputs.network }}-canopy" --source-disk-zone="$ZONE" + gcloud compute images create "zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" --source-disk="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" --source-disk-zone="${{ env.ZONE }}" # Clean up - name: Delete test instance # Always run even if the earlier step fails if: ${{ always() }} run: | - gcloud compute instances delete "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone "$ZONE" + gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a7df67cf..39dca7992 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,8 +14,10 @@ on: - '.github/workflows/test.yml' env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} ZONE: europe-west1-b + MACHINE_TYPE: n2-standard-8 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: @@ -27,60 +29,44 @@ jobs: with: persist-credentials: false - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - REPOSITORY=${GITHUB_REPOSITORY,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \ - echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 - name: Set up gcloud - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} - # Run once: create firewall rule to allow incoming traffic to the nodes - # - name: Create Zcash incoming traffic firewall rule - # run: | - # gcloud compute firewall-rules create "allow-zcash" \ - # --target-tags zebrad \ - # --allow tcp:8233,tcp:18233 \ - # --source-ranges 0.0.0.0/0 \ - # --description="Allow incoming Zcash traffic from anywhere" \ - # Creates Compute Engine virtual machine instance w/ disks - name: Create instance run: | - gcloud compute instances create-with-container "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" \ + gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 100GB \ --boot-disk-type pd-ssd \ --container-image rust:buster \ - --container-mount-disk mount-path='/mainnet',name="zebrad-cache-$SHORT_SHA-mainnet-canopy" \ + --container-mount-disk mount-path='/mainnet',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy" \ --container-restart-policy never \ - --create-disk name="zebrad-cache-$SHORT_SHA-mainnet-canopy",image=zebrad-cache-1558f3378-mainnet-canopy \ - --machine-type n2-standard-8 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy",image=zebrad-cache-1558f3378-mainnet-canopy \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ --tags zebrad \ - --zone "$ZONE" + --zone "${{ env.ZONE }}" # Build and run test container - name: Run all tests run: | - gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --ssh-flag="-o ServerAliveInterval=5" --zone "$ZONE" --command \ - "git clone -b $BRANCH_NAME https://github.com/ZcashFoundation/zebra.git && + gcloud compute ssh "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" --command \ + "git clone -b ${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} https://github.com/ZcashFoundation/zebra.git && cd zebra/ && - docker build --build-arg SHORT_SHA=$SHORT_SHA -f docker/Dockerfile.test -t zebrad-test . && + docker build --build-arg SHORT_SHA=${{ env.GITHUB_SHA_SHORT }} -f docker/Dockerfile.test -t zebrad-test . && docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 zebrad-test:latest cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored && - docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-$SHORT_SHA-mainnet-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet + docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet " # Clean up - name: Delete test instance # Always run even if the earlier step fails if: ${{ always() }} run: | - gcloud compute instances delete "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone "$ZONE" + gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}" diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml index e56fc89e7..73e1973c9 100644 --- a/.github/workflows/zcashd-manual-deploy.yml +++ b/.github/workflows/zcashd-manual-deploy.yml @@ -9,8 +9,10 @@ on: default: 10 env: - PROJECT_ID: zealous-zebra + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} REGION: us-central1 + MACHINE_TYPE: n2-standard-4 + DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com jobs: @@ -22,31 +24,26 @@ jobs: - uses: actions/checkout@v2.4.0 with: persist-credentials: false - - - name: Set project and image names - run: | - BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 # Setup gcloud CLI - name: Set up gcloud SDK environment - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0.4.0 with: - version: '295.0.0' project_id: ${{ env.PROJECT_ID }} service_account_key: ${{ secrets.GCLOUD_AUTH }} # Create instance template from container image - name: Create instance template run: | - gcloud compute instance-templates create-with-container "zcashd-$BRANCH_NAME-$SHORT_SHA" \ + gcloud compute instance-templates create-with-container "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 100GB \ --container-image "electriccoinco/zcashd" \ --container-env ZCASHD_NETWORK="${{ github.event.inputs.network }}" \ - --machine-type n2-standard-4 \ - --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ --tags zcashd \ @@ -55,16 +52,16 @@ jobs: id: does-group-exist continue-on-error: true run: | - gcloud compute instance-groups list | grep "zcashd-$BRANCH_NAME-${{ github.event.inputs.network }}" | grep "$REGION" + gcloud compute instance-groups list | grep "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ github.event.inputs.network }}" | grep "${{ env.REGION }}" # Deploy new managed instance group using the new instance template - name: Create managed instance group if: steps.does-group-exist.outcome == 'failure' run: | gcloud compute instance-groups managed create \ - "zcashd-$BRANCH_NAME-${{ github.event.inputs.network }}" \ - --template "zcashd-$BRANCH_NAME-$SHORT_SHA" \ - --region "$REGION" \ + "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ github.event.inputs.network }}" \ + --template "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --region "${{ env.REGION }}" \ --size "${{ github.event.inputs.size }}" # Rolls out update to existing group using the new instance template @@ -72,6 +69,6 @@ jobs: if: steps.does-group-exist.outcome == 'success' run: | gcloud compute instance-groups managed rolling-action start-update \ - "zcashd-$BRANCH_NAME-${{ github.event.inputs.network }}" \ - --version template="zcashd-$BRANCH_NAME-$SHORT_SHA" \ - --region "$REGION" + "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ github.event.inputs.network }}" \ + --version template="zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + --region "${{ env.REGION }}"