feat(ci)!: add a reusable workflow for deployable integration tests (#4271)

Co-authored-by: teor <teor@riseup.net>

.github/workflows/gcp-test-deploy.yml

@@ -0,0 +1,256 @@
+name: Test instance deployment
+
+on:
+  workflow_call:
+    inputs:
+      network:
+        required: false
+        type: string
+      app_name:
+        required: false
+        type: string
+        default: 'zebra'
+      test_id:
+        required: true
+        type: string
+      test_description:
+        required: true
+        type: string
+      test_variables:
+        required: true
+        type: string
+      zebra_state_path:
+        required: false
+        type: string
+        default: '/zebrad-cache'
+      disk_prefix:
+        required: false
+        type: string
+        default: 'zebrad-cache'
+      disk_suffix:
+        required: false
+        type: string
+      needs_zebra_state:
+        required: true
+        type: boolean
+        default: false
+      saves_to_disk:
+        required: true
+        type: boolean
+      height_grep_text:
+        required: false
+        type: string
+
+env:
+  NETWORK: Mainnet
+  IMAGE_NAME: zebrad-test
+  GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra
+  ZONE: us-central1-a
+  MACHINE_TYPE: c2d-standard-16
+
+jobs:
+  test-without-cached-state:
+    name: Run ${{ inputs.test_id }} test
+    if: ${{ !inputs.needs_zebra_state }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+        with:
+          short-length: 7
+
+      - name: Downcase network name for disks
+        run: |
+          NETWORK_CAPS=${{ inputs.network }}
+          echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV
+
+      # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.7.1
+        with:
+          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
+          service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
+          token_format: 'access_token'
+
+      - name: Create GCP compute instance
+        id: create-instance
+        run: |
+          gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
+          --boot-disk-size 100GB \
+          --boot-disk-type pd-ssd \
+          --create-disk name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=100GB,type=pd-ssd \
+          --container-image debian:buster \
+          --container-restart-policy=never \
+          --machine-type ${{ env.MACHINE_TYPE }} \
+          --scopes cloud-platform \
+          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
+          --tags ${{ inputs.app_name }} \
+          --zone ${{ env.ZONE }}
+          sleep 60
+
+      - name: Run ${{ inputs.test_id }} test
+        run: |
+          gcloud compute ssh \
+          ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
+          --zone ${{ env.ZONE }} \
+          --quiet \
+          --ssh-flag="-o ServerAliveInterval=5" \
+          --command \
+          "\
+          sudo mkfs.ext4 /dev/sdb \
+          && \
+          docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \
+          ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \
+          && \
+          docker run ${{ inputs.test_variables }} -t --name ${{ inputs.test_id }} \
+          --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.zebra_state_path }} \
+          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}"
+
+      - name: Get state version from constants.rs
+        run: |
+          LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" $GITHUB_WORKSPACE/zebra-state/src/constants.rs | grep -oE "[0-9]+" | tail -n1)
+          echo "STATE_VERSION: $LOCAL_STATE_VERSION"
+
+          echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV
+
+      - name: Get sync height from logs
+        run: |
+          SYNC_HEIGHT=""
+
+          DOCKER_LOGS=$(\
+          gcloud compute ssh \
+          ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
+          --zone ${{ env.ZONE }} \
+          --quiet \
+          --ssh-flag="-o ServerAliveInterval=5" \
+          --command="docker logs ${{ inputs.test_id }} --tail 20")
+
+          SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE '${{ inputs.height_grep_text }}\([0-9]+\)' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]])
+          echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV
+
+      # Create image from disk that will be used for following tests
+      # Force the image creation as the disk is still attached even though is not being used by the container
+      - name: Create image from state disk
+        if: ${{ inputs.saves_to_disk }}
+        run: |
+          gcloud compute images create ${{ inputs.disk_prefix }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-${{ inputs.disk_suffix }} \
+          --force \
+          --source-disk=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \
+          --source-disk-zone=${{ env.ZONE }} \
+          --storage-location=us \
+          --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}"
+
+      - name: Delete test instance
+        # If the disk generation step timeouts (+6 hours) the previous step (creating the image) willl be skipped.
+        # Even if the instance continues running, no image will be created, so it's better to delete it.
+        if: always()
+        continue-on-error: true
+        run: |
+          INSTANCE=$(gcloud compute instances list --filter=${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
+          if [ -z "${INSTANCE}" ]; then
+            echo "No instance to delete"
+          else
+            gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet
+          fi
+
+  test-with-cached-state:
+    name: Run ${{ inputs.test_id }} test
+    if: ${{ inputs.needs_zebra_state }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - uses: actions/checkout@v3.0.2
+        with:
+          persist-credentials: false
+          fetch-depth: '2'
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+        with:
+          short-length: 7
+
+      - name: Downcase network name for disks
+        run: |
+          NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }}
+          echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV
+
+      # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.7.1
+        with:
+          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
+          service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
+          token_format: 'access_token'
+
+      # Before executing any further steps, validate the local state and remote version are the same,
+      # or at least that the local state version is greater than the available cached state version from main.
+      #
+      # Aftwards, get the disk name to be used on further steps
+      - name: Validate local state version with cached state
+        id: get-disk-name
+        run: |
+          LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1)
+          echo "LOCAL_STATE_VERSION: $LOCAL_STATE_VERSION"
+
+          ZEBRA_STATE_DISK=$(gcloud compute images list --filter="name~${{ inputs.disk_prefix }} AND name~-${{ inputs.disk_suffix }}" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
+          echo "Disk: $ZEBRA_STATE_DISK"
+          echo "Description: $(gcloud compute images describe $ZEBRA_STATE_DISK --format='value(DESCRIPTION)')"
+
+          GCP_STATE_VERSION=$(echo "$ZEBRA_STATE_DISK" | grep -oE "v[0-9]+" | grep -oE "[0-9]+")
+          echo "GCP_STATE_VERSION: $GCP_STATE_VERSION"
+
+          if [[ "$LOCAL_STATE_VERSION" -lt "$GCP_STATE_VERSION" ]]; then echo "Local version is lower than cached version" && exit 1; fi
+
+          echo "ZEBRA_CACHED_DISK_NAME=$ZEBRA_STATE_DISK" >> $GITHUB_ENV
+
+      # Creates Compute Engine virtual machine instance w/ disks
+      - name: Create GCP compute instance
+        id: create-instance
+        run: |
+          gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
+          --boot-disk-size 100GB \
+          --boot-disk-type pd-ssd \
+          --create-disk image=${{ env.ZEBRA_CACHED_DISK_NAME }},name="${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=100GB,type=pd-ssd \
+          --container-image debian:buster \
+          --container-restart-policy=never \
+          --machine-type ${{ env.MACHINE_TYPE }} \
+          --scopes cloud-platform \
+          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
+          --tags ${{ inputs.app_name }} \
+          --zone ${{ env.ZONE }}
+          sleep 60
+
+      - name: Run ${{ inputs.test_id }} test
+        run: |
+          gcloud compute ssh \
+          ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
+          --zone ${{ env.ZONE }} \
+          --quiet \
+          --ssh-flag="-o ServerAliveInterval=5" \
+          --command \
+          "\
+          docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \
+          ${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \
+          && \
+          docker run ${{ inputs.test_variables }} -t --name ${{ inputs.test_id }} \
+          --mount type=volume,src=${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.zebra_state_path }} \
+          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}"
+
+      - name: Delete test instance
+        # We don't want to leave a failed instance in GCP using resources
+        if: always()
+        continue-on-error: true
+        run: |
+          INSTANCE=$(gcloud compute instances list --filter=${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
+          if [ -z "${INSTANCE}" ]; then
+            echo "No instance to delete"
+          else
+            gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet
+          fi

.github/workflows/lint.yml

@@ -107,3 +107,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3.0.2
       - uses: reviewdog/action-actionlint@v1.24.0
+        with:
+          reporter: github-check
+          fail_on_error: false

.github/workflows/test-full-sync.yml

@@ -1,194 +0,0 @@
-name: Full sync test
-
-on:
-  workflow_dispatch:
-    inputs:
-      network:
-        default: 'Mainnet'
-        description: 'Network to deploy: Mainnet or Testnet'
-        required: true
-      checkpoint_sync:
-        default: 'true'
-        description: 'Configures `zebrad` to use as many checkpoints as possible'
-        required: true
-  pull_request:
-    branches:
-      - main
-    paths:
-      # code and tests (including full sync acceptance test changes)
-      # TODO: ignore changes in test code that isn't used in the full sync test
-      - '**/*.rs'
-      # hard-coded checkpoints
-      # TODO: ignore changes to proptest seed .txt files
-      - '**/*.txt'
-      # dependencies
-      - '**/Cargo.toml'
-      - '**/Cargo.lock'
-      # workflow definitions
-      - 'docker/**'
-      - '.github/workflows/test-full-sync.yml'
-  push:
-    branches:
-      - main
-    paths:
-      # code and tests (including full sync acceptance test changes)
-      # TODO: ignore changes in test code that isn't used in the full sync test
-      - '**/*.rs'
-      # hard-coded checkpoints
-      # TODO: ignore changes to proptest seed .txt files
-      - '**/*.txt'
-      # dependencies
-      - '**/Cargo.toml'
-      - '**/Cargo.lock'
-      # workflow definitions
-      - 'docker/**'
-      - '.github/workflows/test-full-sync.yml'
-      - '.github/workflows/docker-image-build.yml'
-
-env:
-  NETWORK: Mainnet
-  PROJECT_ID: zealous-zebra
-  IMAGE_NAME: zebrad-test
-  GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra
-  REGION: us-central1
-  ZONE: us-central1-a
-  MACHINE_TYPE: c2d-standard-16
-
-jobs:
-  build:
-    # TODO add `startsWith(github.head_ref, 'mergify/merge-queue/')` to the condition to
-    # only run on Mergify head branches, and on manual dispatch:
-    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1
-    if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
-    uses: ./.github/workflows/docker-image-build.yml
-    with:
-      dockerfile_path: ./docker/Dockerfile
-      dockerfile_target: tester
-      image_name: zebrad-test
-      network: Mainnet
-      checkpoint_sync: true
-      rust_backtrace: full
-      rust_lib_backtrace: full
-      colorbt_show_hidden: '1'
-      zebra_skip_ipv6_tests: '1'
-      rust_log: info
-
-  # Test that Zebra can run a full mainnet sync after a PR is approved
-  test-full-sync:
-    name: Test full Mainnet sync
-    runs-on: ubuntu-latest
-    needs: [build]
-    permissions:
-      contents: 'read'
-      id-token: 'write'
-    steps:
-      - uses: actions/checkout@v3.0.2
-        with:
-          persist-credentials: false
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-        with:
-          short-length: 7
-
-      - name: Downcase network name for disks
-        run: |
-          NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }}
-          echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV
-
-      # Setup gcloud CLI
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@v0.7.1
-        with:
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
-          token_format: 'access_token'
-
-      # Creates Compute Engine virtual machine instance w/ disks
-      - name: Create GCP compute instance
-        run: |
-          gcloud compute instances create-with-container "full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --boot-disk-size 100GB \
-          --boot-disk-type pd-ssd \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip",device-name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip",size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path="/zebrad-cache",name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip" \
-          --container-image debian:buster \
-          --container-restart-policy=never \
-          --machine-type ${{ env.MACHINE_TYPE }} \
-          --scopes cloud-platform \
-          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
-          --tags zebrad \
-          --zone "${{ env.ZONE }}"
-          sleep 60
-
-      - name: Full sync
-        id: full-sync
-        run: |
-          gcloud compute ssh \
-          full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command \
-          "docker run -e TEST_FULL_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 -t --name full-sync \
-          --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip,target=/zebrad-cache \
-          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}"
-
-          EXIT_CODE=$(\
-          gcloud compute ssh \
-          full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker wait full-sync")
-
-          exit ${EXIT_CODE}
-
-      - name: Get state version from constants.rs
-        run: |
-          STATE_VERSION=""
-
-          LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" $GITHUB_WORKSPACE/zebra-state/src/constants.rs | grep -oE "[0-9]+" | tail -n1)
-          echo "STATE_VERSION: $LOCAL_STATE_VERSION"
-
-          echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV
-
-      - name: Get sync height from logs
-        run: |
-          SYNC_HEIGHT=""
-
-          DOCKER_LOGS=$(\
-          gcloud compute ssh \
-          full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker logs full-sync --tail 20")
-
-          SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE 'finished initial sync to chain tip, using gossiped blocks sync_percent=100.000 % current_height=Height\([0-9]+\)' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]])
-          echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV
-
-      # Create image from disk
-      # Force the image creation as the disk is still attached, even though is not being used by the container
-      - name: Create image from state disk
-        run: |
-          gcloud compute images create zebrad-cache-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-tip \
-          --force \
-          --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip \
-          --source-disk-zone=${{ env.ZONE }} \
-          --storage-location=us \
-          --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}"
-
-      - name: Delete test instance
-        # If the `full-sync` step timeouts (+6 hours) the previous step (creating the image) willl be skipped.
-        # Even if the instance continues running, no image will be created, so it's better to delete it.
-        if: always()
-        continue-on-error: true
-        run: |
-          INSTANCE=$(gcloud compute instances list --filter=full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
-          if [ -z "${INSTANCE}" ]; then
-            echo "No instance to delete"
-          else
-            gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet
-          fi

.github/workflows/test.patch.yml

@@ -5,13 +5,38 @@ on:
     branches:
       - main
     paths-ignore:
+      # code and tests
       - '**/*.rs'
+      # hard-coded checkpoints and proptest regressions
       - '**/*.txt'
+      # test data snapshots
       - '**/*.snap'
+      # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # workflow definitions
       - 'docker/**'
       - '.github/workflows/test.yml'
+      - '.github/workflows/gcp-test-deploy.yml'
+      - '.github/workflows/docker-image-build.yml'
+
+  push:
+    branches:
+      - main
+    paths-ignore:
+      # code and tests
+      - '**/*.rs'
+      # hard-coded checkpoints and proptest regressions
+      - '**/*.txt'
+      # test data snapshots
+      - '**/*.snap'
+      # dependencies
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      # workflow definitions
+      - 'docker/**'
+      - '.github/workflows/test.yml'
+      - '.github/workflows/gcp-test-deploy.yml'
       - '.github/workflows/docker-image-build.yml'
 
 jobs:
@@ -65,3 +90,11 @@ jobs:
     needs: [build, regenerate-stateful-disks]
     steps:
       - run: 'echo "No build required"'
+
+  # Test that Zebra can run a full mainnet sync after a PR is approved
+  test-full-sync:
+    name: Test a full sync up to the tip
+    runs-on: ubuntu-latest
+    needs: [build, regenerate-stateful-disks]
+    steps:
+      - run: 'echo "No build required"'

.github/workflows/test.yml

@@ -16,6 +16,12 @@ on:
         default: false
         description: 'Just update stateful disks'
         required: true
+      run-full-sync:
+        type: boolean
+        default: false
+        description: 'Just trigger a full sync'
+        required: true
+
   pull_request:
     branches:
       - main
@@ -32,6 +38,8 @@ on:
       # workflow definitions
       - 'docker/**'
       - '.github/workflows/test.yml'
+      - '.github/workflows/gcp-test-deploy.yml'
+      - '.github/workflows/docker-image-build.yml'
 
   push:
     branches:
@@ -49,24 +57,37 @@ on:
       # workflow definitions
       - 'docker/**'
       - '.github/workflows/test.yml'
+      - '.github/workflows/gcp-test-deploy.yml'
       - '.github/workflows/docker-image-build.yml'
 
 env:
-  NETWORK: Mainnet
+  # TODO: use the output from ./.github/workflows/docker-image-build.yml
-  PROJECT_ID: zealous-zebra
   IMAGE_NAME: zebrad-test
   GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra
-  REGION: us-central1
-  ZONE: us-central1-a
-  MACHINE_TYPE: c2d-standard-4
 
 jobs:
+  changed-files:
+    runs-on: ubuntu-latest
+    name: Test changed-files
+    outputs:
+      modified_files: ${{ steps.changed-files.outputs.modified_files }}
+    steps:
+      - uses: actions/checkout@v3.0.2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v19
+
   build:
     uses: ./.github/workflows/docker-image-build.yml
     with:
       dockerfile_path: ./docker/Dockerfile
       dockerfile_target: tester
       image_name: zebrad-test
+      # TODO: validate how to use variable/conditional values for Testnet
       network: Mainnet
       checkpoint_sync: true
       rust_backtrace: full
@@ -132,6 +153,7 @@ jobs:
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
           docker run --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
 
+  # Test launching lightwalletd with an empty lightwalletd and Zebra state
   test-lightwalletd-integration:
     name: Test integration with lightwalletd
     runs-on: ubuntu-latest
@@ -143,245 +165,53 @@ jobs:
         with:
           short-length: 7
 
-      - name: Run tests with included lightwalletd binary
+      - name: Run tests with empty lightwalletd launch
         run: |
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_TEST_LIGHTWALLETD --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance -- lightwalletd_integration --nocapture
+          docker run -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance -- lightwalletd_integration --nocapture
         env:
           ZEBRA_TEST_LIGHTWALLETD: '1'
 
   regenerate-stateful-disks:
-    name: Regenerate stateful disks
+    needs: [ build, changed-files]
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/gcp-test-deploy.yml
-    needs: build
+    # Only run this job if the database format version has (likely) changed.
-    outputs:
+    if: ${{ (contains(needs.changed-files.outputs.modified_files, 'zebra-state/src/constants.rs') && github.event_name == 'push') || github.event.inputs.regenerate-disks == 'true' }}
-      any_changed: ${{ steps.changed-files-specific.outputs.any_changed }}
+    with:
-    permissions:
+      test_id: sync-to-checkpoint
-      contents: 'read'
+      test_description: Test sync up to mandatory checkpoint
-      id-token: 'write'
+      test_variables: '-e TEST_DISK_REBUILD=1 -e ZEBRA_FORCE_USE_COLOR=1'
-    steps:
+      needs_zebra_state: false
-      - uses: actions/checkout@v3.0.2
+      saves_to_disk: true
-        with:
+      disk_suffix: checkpoint
-          persist-credentials: false
+      height_grep_text: 'flushing database to disk height=Height'
-          fetch-depth: '2'
-
-      # TODO move the `changed-files-specific` step to the build job for a better dependency tree
-      # Only run this job if the database format version has (likely) changed.
-      #
-      # If we have accidentally changed the format, but not changed the version,
-      # we want to run with the old cached state, so this job fails.
-      #
-      # If we change the state path without changing the version,
-      # this job will take a few hours, because it will do a full rebuild.
-      - name: Get specific changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v19
-        with:
-          files: |
-            zebra-state/**/constants.rs
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-        with:
-          short-length: 7
-
-      - name: Downcase network name for disks
-        run: |
-          NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }}
-          echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV
-
-      # Setup gcloud CLI
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@v0.7.1
-        with:
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
-          token_format: 'access_token'
-
-      - name: Create GCP compute instance
-        id: create-instance
-        if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' || github.event_name == 'push'}}
-        run: |
-          gcloud compute instances create-with-container "sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --boot-disk-size 100GB \
-          --boot-disk-type pd-ssd \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint",device-name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint",size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path="/zebrad-cache",name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint" \
-          --container-image debian:buster \
-          --container-restart-policy=never \
-          --machine-type ${{ env.MACHINE_TYPE }} \
-          --scopes cloud-platform \
-          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
-          --tags zebrad \
-          --zone "${{ env.ZONE }}"
-          sleep 60
-
-      - name: Regenerate stateful disks
-        id: sync-to-checkpoint
-        if: ${{ steps.create-instance.outcome == 'success' }}
-        run: |
-          gcloud compute ssh \
-          sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command \
-          "docker run -t --name sync-to-checkpoint \
-          --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint,target=/zebrad-cache \
-          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \
-          cargo test --locked --release --features enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ env.NETWORK }}"
-
-          EXIT_CODE=$(\
-          gcloud compute ssh \
-          sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker wait sync-to-checkpoint")
-
-          exit ${EXIT_CODE}
-
-      - name: Get state version from constants.rs
-        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }}
-        run: |
-          STATE_VERSION=""
-
-          LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" $GITHUB_WORKSPACE/zebra-state/src/constants.rs | grep -oE "[0-9]+" | tail -n1)
-          echo "STATE_VERSION: $LOCAL_STATE_VERSION"
-
-          echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV
-
-      - name: Get sync height from logs
-        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }}
-        run: |
-          SYNC_HEIGHT=""
-
-          DOCKER_LOGS=$(\
-          gcloud compute ssh \
-          sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker logs sync-to-checkpoint --tail 20")
-
-          SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE 'flushing database to disk height=Height\([0-9]+\)' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]])
-
-          echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV
-
-      # Create image from disk that will be used for the sync past mandatory checkpoint test
-      # Force the image creation as the disk is still attached even though is not being used by the container
-      - name: Create image from state disk
-        # Only run if the earlier step succeeds
-        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }}
-        run: |
-          gcloud compute images create zebrad-cache-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-checkpoint \
-          --force \
-          --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint \
-          --source-disk-zone=${{ env.ZONE }} \
-          --storage-location=us \
-          --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}"
-
-      - name: Delete test instance
-        # If the `sync-to-checkpoint` step timeouts (+6 hours) the previous step (creating the image) willl be skipped.
-        # Even if the instance continues running, no image will be created, so it's better to delete it.
-        if: always()
-        continue-on-error: true
-        run: |
-          INSTANCE=$(gcloud compute instances list --filter=sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
-          if [ -z "${INSTANCE}" ]; then
-            echo "No instance to delete"
-          else
-            gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet
-          fi
 
   # Test that Zebra syncs and fully validates a few thousand blocks from a cached post-checkpoint state
   test-stateful-sync:
-    name: Test full validation sync from cached state
+    needs: build
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/gcp-test-deploy.yml
-    needs: [build, regenerate-stateful-disks]
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
-    permissions:
+    with:
-      contents: 'read'
+      test_id: sync-past-checkpoint
-      id-token: 'write'
+      test_description: Test full validation sync from a cached state
-    steps:
+      test_variables: '-e TEST_CHECKPOINT_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1'
-      - name: Inject slug/short variables
+      needs_zebra_state: true
-        uses: rlespinasse/github-slug-action@v4
+      saves_to_disk: false
-        with:
+      disk_suffix: checkpoint
-          short-length: 7
 
-      - name: Downcase network name for disks
+  # Test that Zebra can run a full mainnet sync after a PR is approved
-        run: |
+  test-full-sync:
-          NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }}
+    needs: build
-          echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV
+    uses: ./.github/workflows/gcp-test-deploy.yml
-
+    # TODO change `github.ref_name == 'main'` to `startsWith(github.head_ref, 'mergify/merge-queue/')`
-      # Setup gcloud CLI
+    # to only run on Mergify head branches, and on manual dispatch:
-      - name: Authenticate to Google Cloud
+    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1
-        id: auth
+    if: ${{ (github.event_name == 'push' && github.ref_name == 'main') || github.event.inputs.run-full-sync == 'true' }}
-        uses: google-github-actions/auth@v0.7.1
+    with:
-        with:
+      test_id: full-sync-to-tip
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
+      test_description: Test a full sync up to the tip
-          service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
+      test_variables: '-e TEST_FULL_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600'
-          token_format: 'access_token'
+      needs_zebra_state: false
-
+      saves_to_disk: true
-      - name: Get disk state name from gcloud
+      disk_suffix: tip
-        id: get-disk-name
+      height_grep_text: 'finished initial sync to chain tip, using gossiped blocks sync_percent=100.* current_height=Height'
-        run: |
-          GCP_STATE_DISK=$(gcloud compute images list --filter="name~zebrad-cache AND name~-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
-          echo "Disk: $GCP_STATE_DISK"
-          echo "Description: $(gcloud compute images describe $GCP_STATE_DISK --format='value(DESCRIPTION)')"
-
-          echo "CACHED_DISK_NAME=$GCP_STATE_DISK" >> $GITHUB_ENV
-
-      # Creates Compute Engine virtual machine instance w/ disks
-      - name: Create GCP compute instance
-        id: create-instance
-        run: |
-          gcloud compute instances create-with-container "sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --boot-disk-size 100GB \
-          --boot-disk-type pd-ssd \
-          --create-disk image=${{ env.CACHED_DISK_NAME }},name="${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }}",size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path="/zebrad-cache",name="${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --container-image debian:buster \
-          --container-restart-policy=never \
-          --machine-type ${{ env.MACHINE_TYPE }} \
-          --scopes cloud-platform \
-          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
-          --tags zebrad \
-          --zone "${{ env.ZONE }}"
-          sleep 60
-
-      - name: Sync past mandatory checkpoint
-        id: sync-past-checkpoint
-        run: |
-          gcloud compute ssh \
-          sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command \
-          "docker run -t --name sync-past-checkpoint \
-          --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }},target=/zebrad-cache \
-          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \
-          cargo test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ env.NETWORK }}"
-
-          EXIT_CODE=$(\
-          gcloud compute ssh \
-          sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker wait sync-past-checkpoint")
-
-          exit ${EXIT_CODE}
-
-      - name: Delete test instance
-        # We don't want to leave a failed instance in GCP using resources
-        if: always()
-        continue-on-error: true
-        run: |
-          INSTANCE=$(gcloud compute instances list --filter=sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
-          if [ -z "${INSTANCE}" ]; then
-            echo "No instance to delete"
-          else
-            gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet
-          fi

docker/entrypoint.sh

@@ -23,6 +23,10 @@ case "$1" in
             exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--workspace" "--" "--include-ignored"
         elif [[ "$TEST_FULL_SYNC" -eq "1" ]]; then
             exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--test" "acceptance" "--" "--nocapture" "--ignored" "full_sync_mainnet"
+        elif [[ "$TEST_DISK_REBUILD" -eq "1" ]]; then
+            exec cargo "test" "--locked" "--release" "--features" "enable-sentry,test_sync_to_mandatory_checkpoint_${NETWORK,,}" "--manifest-path" "zebrad/Cargo.toml" "sync_to_mandatory_checkpoint_${NETWORK,,}"
+        elif [[ "$TEST_CHECKPOINT_SYNC" -eq "1" ]]; then
+            exec cargo "test" "--locked" "--release" "--features" "enable-sentry,test_sync_past_mandatory_checkpoint_${NETWORK,,}" "--manifest-path" "zebrad/Cargo.toml" "sync_past_mandatory_checkpoint_${NETWORK,,}"
         else
             exec "$@"
         fi