zebra/.github/workflows/test.yml

name: Test

on:
  workflow_dispatch:
    inputs:
      network:
        default: 'Mainnet'
        description: 'Network to deploy: Mainnet or Testnet'
        required: true
      checkpoint_sync:
        default: 'true'
        description: 'Configures `zebrad` to use as many checkpoints as possible'
        required: true
      regenerate-disks:
        type: boolean
        default: false
        description: 'Just update stateful disks'
        required: true
  pull_request:
    branches:
      - main
    paths:
      # code and tests
      - '**/*.rs'
      # hard-coded checkpoints and proptest regressions
      - '**/*.txt'
      # test data snapshots
      - '**/*.snap'
      # dependencies
      - '**/Cargo.toml'
      - '**/Cargo.lock'
      # workflow definitions
      - 'docker/**'
      - '.github/workflows/test.yml'

  push:
    branches:
      - main
    paths:
      # code and tests
      - '**/*.rs'
      # hard-coded checkpoints and proptest regressions
      - '**/*.txt'
      # test data snapshots
      - '**/*.snap'
      # dependencies
      - '**/Cargo.toml'
      - '**/Cargo.lock'
      # workflow definitions
      - 'docker/**'
      - '.github/workflows/test.yml'
      - '.github/workflows/docker-image-build.yml'

env:
  NETWORK: Mainnet
  PROJECT_ID: zealous-zebra
  IMAGE_NAME: zebrad-test
  GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra
  REGION: us-central1
  ZONE: us-central1-a
  MACHINE_TYPE: c2d-standard-4

jobs:
  build:
    uses: ./.github/workflows/docker-image-build.yml
    with:
      dockerfile_path: ./docker/Dockerfile
      dockerfile_target: tester
      image_name: zebrad-test
      network: Mainnet
      checkpoint_sync: true
      rust_backtrace: full
      rust_lib_backtrace: full
      colorbt_show_hidden: '1'
      zebra_skip_ipv6_tests: '1'
      rust_log: info

  # Run all the zebra tests, including tests that are ignored by default
  test-all:
    name: Test all
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run all zebrad tests
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- --include-ignored

  # This test changes zebra-chain's activation heights,
  # which can recompile all the Zebra crates,
  # so we want its build products to be cached separately.
  #
  # Also, we don't want to accidentally use the fake heights in other tests.
  test-fake-activation-heights:
    name: Test with fake activation heights
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run tests with fake activation heights
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights
        env:
          TEST_FAKE_ACTIVATION_HEIGHTS: '1'

  # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state
  test-empty-sync:
    name: Test checkpoint sync from empty state
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run zebrad large sync tests
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored

  test-lightwalletd-integration:
    name: Test integration with lightwalletd
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run tests with included lightwalletd binary
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run -e ZEBRA_TEST_LIGHTWALLETD --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance -- lightwalletd_integration --nocapture
        env:
          ZEBRA_TEST_LIGHTWALLETD: '1'

  regenerate-stateful-disks:
    name: Regenerate stateful disks
    runs-on: ubuntu-latest
    needs: build
    outputs:
      any_changed: ${{ steps.changed-files-specific.outputs.any_changed }}
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - uses: actions/checkout@v3.0.2
        with:
          persist-credentials: false
          fetch-depth: '2'

      # TODO move the `changed-files-specific` step to the build job for a better dependency tree
      # Only run this job if the database format version has (likely) changed.
      #
      # If we have accidentally changed the format, but not changed the version,
      # we want to run with the old cached state, so this job fails.
      #
      # If we change the state path without changing the version,
      # this job will take a few hours, because it will do a full rebuild.
      - name: Get specific changed files
        id: changed-files-specific
        uses: tj-actions/changed-files@v19
        with:
          files: |
            zebra-state/**/constants.rs

      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Downcase network name for disks
        run: |
          NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }}
          echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV

      # Setup gcloud CLI
      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@v0.7.1
        with:
          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
          service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
          token_format: 'access_token'

      - name: Create GCP compute instance
        id: create-instance
        if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' || github.event_name == 'push'}}
        run: |
          gcloud compute instances create-with-container "sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
          --boot-disk-size 100GB \
          --boot-disk-type pd-ssd \
          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint",device-name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint",size=100GB,type=pd-ssd \
          --container-mount-disk mount-path="/zebrad-cache",name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint" \
          --container-image debian:buster \
          --container-restart-policy=never \
          --machine-type ${{ env.MACHINE_TYPE }} \
          --scopes cloud-platform \
          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
          --tags zebrad \
          --zone "${{ env.ZONE }}"
          sleep 60

      - name: Regenerate stateful disks
        id: sync-to-checkpoint
        if: ${{ steps.create-instance.outcome == 'success' }}
        run: |
          gcloud compute ssh \
          sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
          --zone ${{ env.ZONE }} \
          --quiet \
          --ssh-flag="-o ServerAliveInterval=5" \
          --command \
          "docker run -t --name sync-to-checkpoint \
          --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint,target=/zebrad-cache \
          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \
          cargo test --locked --release --features enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ env.NETWORK }}"

          EXIT_CODE=$(\
          gcloud compute ssh \
          sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
          --zone ${{ env.ZONE }} \
          --quiet \
          --ssh-flag="-o ServerAliveInterval=5" \
          --command="docker wait sync-to-checkpoint")

          exit ${EXIT_CODE}

      - name: Get state version from constants.rs
        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }}
        run: |
          STATE_VERSION=""

          LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" $GITHUB_WORKSPACE/zebra-state/src/constants.rs | grep -oE "[0-9]+" | tail -n1)
          echo "STATE_VERSION: $LOCAL_STATE_VERSION"

          echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV

      - name: Get sync height from logs
        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }}
        run: |
          SYNC_HEIGHT=""

          DOCKER_LOGS=$(\
          gcloud compute ssh \
          sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
          --zone ${{ env.ZONE }} \
          --quiet \
          --ssh-flag="-o ServerAliveInterval=5" \
          --command="docker logs sync-to-checkpoint --tail 20")

          SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE 'flushing database to disk height=Height\([0-9]+\)' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]])

          echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV

      # Create image from disk that will be used for the sync past mandatory checkpoint test
      # Force the image creation as the disk is still attached even though is not being used by the container
      - name: Create image from state disk
        # Only run if the earlier step succeeds
        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }}
        run: |
          gcloud compute images create zebrad-cache-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-checkpoint \
          --force \
          --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint \
          --source-disk-zone=${{ env.ZONE }} \
          --storage-location=us \
          --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}"

      - name: Delete test instance
        # If the `sync-to-checkpoint` step timeouts (+6 hours) the previous step (creating the image) willl be skipped.
        # Even if the instance continues running, no image will be created, so it's better to delete it.
        if: always()
        continue-on-error: true
        run: |
          INSTANCE=$(gcloud compute instances list --filter=sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
          if [ -z "${INSTANCE}" ]; then
            echo "No instance to delete"
          else
            gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet
          fi

  # Test that Zebra syncs and fully validates a few thousand blocks from a cached post-checkpoint state
  test-stateful-sync:
    name: Test full validation sync from cached state
    runs-on: ubuntu-latest
    needs: [build, regenerate-stateful-disks]
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Downcase network name for disks
        run: |
          NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }}
          echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV

      # Setup gcloud CLI
      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@v0.7.1
        with:
          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
          service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
          token_format: 'access_token'

      - name: Get disk state name from gcloud
        id: get-disk-name
        run: |
          GCP_STATE_DISK=$(gcloud compute images list --filter="name~zebrad-cache AND name~-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
          echo "Disk: $GCP_STATE_DISK"
          echo "Description: $(gcloud compute images describe $GCP_STATE_DISK --format='value(DESCRIPTION)')"

          echo "CACHED_DISK_NAME=$GCP_STATE_DISK" >> $GITHUB_ENV

      # Creates Compute Engine virtual machine instance w/ disks
      - name: Create GCP compute instance
        id: create-instance
        run: |
          gcloud compute instances create-with-container "sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
          --boot-disk-size 100GB \
          --boot-disk-type pd-ssd \
          --create-disk image=${{ env.CACHED_DISK_NAME }},name="${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }}",size=100GB,type=pd-ssd \
          --container-mount-disk mount-path="/zebrad-cache",name="${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }}" \
          --container-image debian:buster \
          --container-restart-policy=never \
          --machine-type ${{ env.MACHINE_TYPE }} \
          --scopes cloud-platform \
          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
          --tags zebrad \
          --zone "${{ env.ZONE }}"
          sleep 60

      - name: Sync past mandatory checkpoint
        id: sync-past-checkpoint
        run: |
          gcloud compute ssh \
          sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
          --zone ${{ env.ZONE }} \
          --quiet \
          --ssh-flag="-o ServerAliveInterval=5" \
          --command \
          "docker run -t --name sync-past-checkpoint \
          --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }},target=/zebrad-cache \
          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \
          cargo test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ env.NETWORK }}"

          EXIT_CODE=$(\
          gcloud compute ssh \
          sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
          --zone ${{ env.ZONE }} \
          --quiet \
          --ssh-flag="-o ServerAliveInterval=5" \
          --command="docker wait sync-past-checkpoint")

          exit ${EXIT_CODE}

      - name: Delete test instance
        # We don't want to leave a failed instance in GCP using resources
        if: always()
        continue-on-error: true
        run: |
          INSTANCE=$(gcloud compute instances list --filter=sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
          if [ -z "${INSTANCE}" ]; then
            echo "No instance to delete"
          else
            gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet
          fi