zebra/.github/workflows/test.yml

name: Test

on:
  workflow_dispatch:
    inputs:
      network:
        default: 'Mainnet'
      regenerate-disks:
        type: boolean
        default: false
        description: Just update stateful disks
  pull_request:
    branches:
      - main
    paths:
      # code and tests
      - '**/*.rs'
      # hard-coded checkpoints
      - '**/*.txt'
      # test data snapshots
      - '**/*.snap'
      # dependencies
      - '**/Cargo.toml'
      - '**/Cargo.lock'
      # workflow definitions
      - 'docker/**'
      - '.github/workflows/test.yml'
  pull_request_review:
    branches:
      - main
    paths:
      - '**/*.rs'
      - '**/*.txt'
      - '**/Cargo.toml'
      - '**/Cargo.lock'
      - 'docker/**'
      - '.github/workflows/test.yml'
    types: [submitted]

env:
  CARGO_INCREMENTAL: '1'
  ZEBRA_SKIP_IPV6_TESTS: "1"
  NETWORK: Mainnet
  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
  GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
  GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
  REGION: us-central1
  ZONE: us-central1-a
  MACHINE_TYPE: c2d-standard-16
  IMAGE_NAME: zebrad-test

jobs:
  build:
    name: Build images
    timeout-minutes: 210
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2.4.0
        with:
          persist-credentials: false

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4

      # Setup Docker Buildx to allow use of docker cache layers from GH
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1

      - name: Login to Google Artifact Registry
        uses: docker/login-action@v1.12.0
        with:
          registry: us-docker.pkg.dev
          username: _json_key
          password: ${{ secrets.GOOGLE_CREDENTIALS }}

      - name: Login to Google Container Registry
        uses: docker/login-action@v1.12.0
        with:
          registry: gcr.io
          username: _json_key
          password: ${{ secrets.GOOGLE_CREDENTIALS }}

      # Build and push image to Google Artifact Registry
      - name: Build & push
        id: docker_build
        uses: docker/build-push-action@v2.8.0
        with:
          target: tester
          context: .
          file: ./docker/Dockerfile
          tags: |
            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
          build-args: |
            NETWORK=${{ github.event.inputs.network || env.NETWORK }}
            SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
            RUST_BACKTRACE=full
            ZEBRA_SKIP_NETWORK_TESTS="1"
            CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
            RUST_LOG=debug
            SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
          push: true
          cache-from: type=gha
          cache-to: type=gha,mode=max

  # Run all the zebra tests, including tests that are ignored by default
  test-all:
    name: Test all
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
    steps:
      - uses: actions/checkout@v2.4.0
        with:
          persist-credentials: false

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4

      - name: Run all zebrad tests
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- --include-ignored

  test-fake-activation-heights:
    name: Test with fake activation heights
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
    steps:
      - uses: actions/checkout@v2.4.0
        with:
          persist-credentials: false

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4

      - name: Run tests with fake activation heights
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights

  # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state
  test-empty-sync:
    name: Test checkpoint sync from empty state
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
    steps:
      - uses: actions/checkout@v2.4.0
        with:
          persist-credentials: false

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4

      - name: Run zebrad large sync tests
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored

  test-lightwalletd-integration:
    name: Test integration with lightwalletd
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
    steps:
      - uses: actions/checkout@v2.4.0
        with:
          persist-credentials: false

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4

      - name: Run tests with included lightwalletd binary
        run: |
          docker pull ${{ env.GAR_BASE }}/zebrad-test:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
          docker run -e ZEBRA_SKIP_IPV6_TESTS -e ZEBRA_TEST_LIGHTWALLETD --name zebrad-tests -t ${{ env.GAR_BASE }}/zebrad-test:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance -- lightwalletd_integration 
        env:
          ZEBRA_TEST_LIGHTWALLETD: '1'

  regenerate-stateful-disks:
    name: Regenerate stateful disks
    runs-on: ubuntu-latest
    needs: build
    outputs:
      disk_short_sha: ${{ steps.disk-short-sha.outputs.disk_short_sha }}
    steps:
      - uses: actions/checkout@v2.4.0
        with:
          persist-credentials: false
          fetch-depth: '2'

      # only run this job if the database format might have changed
      - name: Get specific changed files
        id: changed-files-specific
        uses: tj-actions/changed-files@v17.2
        with:
          files: |
            /zebra-state/**/config.rs
            /zebra-state/**/constants.rs
            /zebra-state/**/finalized_state.rs
            /zebra-state/**/disk_format.rs
            /zebra-state/**/disk_db.rs
            /zebra-state/**/zebra_db.rs
            /zebra-state/**/zebra_db/block.rs
            /zebra-state/**/zebra_db/chain.rs
            /zebra-state/**/zebra_db/shielded.rs
            /zebra-state/**/zebra_db/transparent.rs

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4

      - name: Downcase network name for disks
        run: |
          echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV

      # Setup gcloud CLI
      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@v0.5.0
        with:
          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}

      - name: Create GCP compute instance
        id: create-instance
        if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' }}
        run: |
          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
          --boot-disk-size 100GB \
          --boot-disk-type pd-ssd \
          --create-disk name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy",size=100GB,type=pd-ssd \
          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy" \
          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
          --container-restart-policy=never \
          --container-stdin \
          --container-tty \
          --container-command="cargo" \
          --container-arg="test" \
          --container-arg="--locked" \
          --container-arg="--release" \
          --container-arg="--features" \
          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
          --container-arg="--manifest-path" \
          --container-arg="zebrad/Cargo.toml" \
          --container-arg="sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
          --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
          --machine-type ${{ env.MACHINE_TYPE }} \
          --scopes cloud-platform \
          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
          --tags zebrad \
          --zone "${{ env.ZONE }}"

      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
      # This TODO relates to the following issues:
      # https://github.com/actions/runner/issues/241
      # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
      - name: Get container name from logs
        id: get-container-name
        if: steps.create-instance.outcome == 'success'
        run: |
          INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
          echo "Using instance: $INSTANCE_ID"
          while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
              echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
              sleep 10
          done
          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
          echo "::set-output name=zebra_container::$CONTAINER_NAME"

      - name: Regenerate stateful disks logs
        id: sync-to-checkpoint
        if: steps.create-instance.outcome == 'success'
        run: |
          gcloud compute ssh \
          zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
          --zone ${{ env.ZONE }} \
          --quiet \
          --ssh-flag="-o ServerAliveInterval=5" \
          --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
        env:
          ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}

      # Create image from disk that will be used to sync past mandatory checkpoint test
      # Force the image creation as the disk is still attached even though is not being used by the container
      - name: Create image from state disk
        # Only run if the earlier step succeeds
        if: steps.sync-to-checkpoint.outcome == 'success'
        run: |
          gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
          --force \
          --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
          --source-disk-zone=${{ env.ZONE }} \
          --storage-location=us \
          --description="Created from head branch ${{ env.GITHUB_HEAD_REF_SLUG_URL }} targeting ${{ env.GITHUB_BASE_REF_SLUG }} from PR ${{ env.GITHUB_REF_SLUG_URL }} with commit ${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA }}"

      - name: Output and write the disk SHORT_SHA to a txt
        id: disk-short-sha
        if: steps.sync-to-checkpoint.outcome == 'success'
        run: |
          short_sha=$(echo "${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}")
          echo "$short_sha" > latest-disk-state-sha.txt
          echo "::set-output name=disk_short_sha::$short_sha"

      - name: Upload the disk state txt
        if: steps.sync-to-checkpoint.outcome == 'success'
        uses: actions/upload-artifact@v2.3.1
        with:
          name: latest-disk-state-sha
          path: latest-disk-state-sha.txt
          retention-days: 1095

      - name: Delete test instance
        # Do not delete the instance if the sync timeouts in GitHub
        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} || ${{ steps.sync-to-checkpoint.outcome == 'failure' }}
        continue-on-error: true
        run: |
          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

  # Test that Zebra syncs and fully validates a few thousand blocks from a cached post-checkpoint state
  test-stateful-sync:
    name: Test full validation sync from cached state
    runs-on: ubuntu-latest
    needs: [ build, regenerate-stateful-disks]
    steps:
      - uses: actions/checkout@v2.4.0
        with:
          persist-credentials: false

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4

      - name: Downcase network name for disks
        run: |
          echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV

      # Get the latest uploaded txt with the disk SHORT_SHA from this workflow
      - name: Download latest disk state SHORT_SHA
        uses: dawidd6/action-download-artifact@v2.17.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          workflow: test.yml
          workflow_conclusion: ''
          name: latest-disk-state-sha
          check_artifacts:  true

      - name: Get disk state SHA from txt
        id: get-disk-sha
        run: |
          output=$(cat latest-disk-state-sha.txt)
          echo "::set-output name=sha::$output"

      # Setup gcloud CLI
      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@v0.5.0
        with:
          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}

      # Creates Compute Engine virtual machine instance w/ disks
      - name: Create GCP compute instance
        id: create-instance
        run: |
          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
          --boot-disk-size 100GB \
          --boot-disk-type pd-ssd \
          --create-disk=image=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy,name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy,size=100GB,type=pd-ssd \
          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
          --container-restart-policy=never \
          --container-stdin \
          --container-tty \
          --container-command="cargo" \
          --container-arg="test" \
          --container-arg="--locked" \
          --container-arg="--release" \
          --container-arg="--features" \
          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
          --container-arg="--manifest-path" \
          --container-arg="zebrad/Cargo.toml" \
          --container-arg="sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
          --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
          --machine-type ${{ env.MACHINE_TYPE }} \
          --scopes cloud-platform \
          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
          --tags zebrad \
          --zone "${{ env.ZONE }}"
        env:
          DISK_SHORT_SHA: ${{ needs.regenerate-stateful-disks.outputs.disk_short_sha || steps.get-disk-sha.outputs.sha }}

      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
      # This TODO relates to the following issues:
      # https://github.com/actions/runner/issues/241
      # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
      - name: Get container name from logs
        id: get-container-name
        if: steps.create-instance.outcome == 'success'
        run: |
          INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
          echo "Using instance: $INSTANCE_ID"
          while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
              echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
              sleep 10
          done
          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
          echo "::set-output name=zebra_container::$CONTAINER_NAME"

      - name: Sync past mandatory checkpoint logs
        id: sync-past-checkpoint
        run: |
          gcloud compute ssh \
          zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
          --zone ${{ env.ZONE }} \
          --quiet \
          --ssh-flag="-o ServerAliveInterval=5" \
          --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
        env:
          ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}

      - name: Delete test instance
        # Do not delete the instance if the sync timeouts in GitHub
        if: ${{ steps.sync-past-checkpoint.outcome == 'success' }} || ${{ steps.sync-past-checkpoint.outcome == 'failure' }}
        continue-on-error: true
        run: |
          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

  # Test that Zebra can run a full mainnet sync after a PR is approved
  test-full-sync:
    name: Test full Mainnet sync
    runs-on: ubuntu-latest
    needs: [ build]
    if: github.event.review.state == 'approved'
    steps:
      - uses: actions/checkout@v2.4.0
        with:
          persist-credentials: false

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4

      # Setup gcloud CLI
      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@v0.5.0
        with:
          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}

      # Creates Compute Engine virtual machine instance w/ disks
      - name: Create GCP compute instance
        id: create-instance
        run: |
          gcloud compute instances create-with-container "sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
          --boot-disk-size 100GB \
          --boot-disk-type pd-extreme \
          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
          --container-restart-policy=never \
          --container-stdin \
          --container-tty \
          --container-env=ZEBRA_SKIP_IPV6_TESTS=1,TEST_FULL_SYNC=1,ZEBRA_FORCE_USE_COLOR=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
          --machine-type ${{ env.MACHINE_TYPE }} \
          --scopes cloud-platform \
          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
          --tags zebrad \
          --zone "${{ env.ZONE }}"

      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
      # This TODO relates to the following issues:
      # https://github.com/actions/runner/issues/241
      # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
      - name: Get container name from logs
        id: get-container-name
        if: steps.create-instance.outcome == 'success'
        run: |
          INSTANCE_ID=$(gcloud compute instances describe sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
          echo "Using instance: $INSTANCE_ID"
          while [[ ${CONTAINER_NAME} != *"sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
              echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
              sleep 10
          done
          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
          echo "::set-output name=zebra_container::$CONTAINER_NAME"

      - name: Sync past mandatory checkpoint logs
        id: sync-past-checkpoint
        run: |
          gcloud compute ssh \
          sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
          --zone ${{ env.ZONE }} \
          --quiet \
          --ssh-flag="-o ServerAliveInterval=5" \
          --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
        env:
          ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}

      - name: Delete test instance
        # Do not delete the instance if the sync timeouts in GitHub
        if: ${{ steps.sync-past-checkpoint.outcome == 'success' }} || ${{ steps.sync-past-checkpoint.outcome == 'failure' }}
        continue-on-error: true
        run: |
          gcloud compute instances delete "sync-tests-${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"