zebra/.github/workflows/ci-integration-tests-gcp.yml

# Google Cloud integration tests that run when Rust code or dependencies are modified,
# but only on PRs from the ZcashFoundation/zebra repository. (External PRs are tested by mergify.)
#
# Specific conditions and dependencies are set for each job to ensure they are executed in the correct sequence and under the right circumstances.
# Each test has a description of the conditions under which it runs.
name: Integration Tests on GCP

# Ensures that only one workflow task will run at a time. Previous builds, if
# already in process, will get cancelled. Only the latest commit will be allowed
# to run, cancelling any workflows in between
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

on:
  schedule:
    # Run this job every Friday at mid-day UTC
    # This is limited to the Zebra and lightwalletd Full Sync jobs
    # TODO: we should move this behavior to a separate workflow
    - cron:  '0 12 * * 5'

  workflow_dispatch:
    inputs:
      network:
        default: 'Mainnet'
        description: 'Network to deploy: Mainnet or Testnet'
        required: true
      regenerate-disks:
        type: boolean
        default: false
        description: 'Just run a Zebra checkpoint sync and update checkpoint disks'
        required: true
      run-full-sync:
        type: boolean
        default: false
        description: 'Just run a Zebra full sync on `network`, and update tip disks'
        required: true
      run-lwd-sync:
        type: boolean
        default: false
        description: 'Just run a lightwalletd full sync and update tip disks'
        required: true
      force_save_to_disk:
        required: false
        type: boolean
        default: false
        description: 'Force tests to always create a cached state disk, if they already create disks'
      no_cache:
        description: 'Disable the Docker cache for this build'
        required: false
        type: boolean
        default: false

  pull_request:
    # Skip PRs where Rust code and dependencies aren't modified.
    paths:
      # code and tests
      - '**/*.rs'
      # hard-coded checkpoints and proptest regressions
      - '**/*.txt'
      # test data snapshots
      - '**/*.snap'
      # dependencies
      - '**/Cargo.toml'
      - '**/Cargo.lock'
      # configuration files
      - '.cargo/config.toml'
      - '**/clippy.toml'
      # workflow definitions
      - 'docker/**'
      - '.github/workflows/ci-integration-tests-gcp.yml'
      - '.github/workflows/sub-deploy-integration-tests-gcp.yml'
      - '.github/workflows/sub-build-docker-image.yml'
      - '.github/workflows/sub-find-cached-disks.yml'

  push:
    # Skip main branch updates where Rust code and dependencies aren't modified.
    branches:
      - main
    paths:
      # code and tests
      - '**/*.rs'
      # hard-coded checkpoints and proptest regressions
      - '**/*.txt'
      # test data snapshots
      - '**/*.snap'
      # dependencies
      - '**/Cargo.toml'
      - '**/Cargo.lock'
      # configuration files
      - '.cargo/config.toml'
      - '**/clippy.toml'
      # workflow definitions
      - 'docker/**'
      - '.dockerignore'
      - '.github/workflows/ci-integration-tests-gcp.yml'
      - '.github/workflows/sub-deploy-integration-tests-gcp.yml'
      - '.github/workflows/sub-find-cached-disks.yml'
      - '.github/workflows/sub-build-docker-image.yml'

# IMPORTANT
#
# The job names in `ci-integration-tests-gcp.yml`, `ci-integration-tests-gcp.patch.yml` and
# `ci-integration-tests-gcp.patch-external.yml` must be kept in sync.
jobs:
  # to also run a job on Mergify head branches,
  # add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`:
  # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1

  # Check if the cached state disks used by the tests are available for the default network.
  #
  # The default network is mainnet unless a manually triggered workflow or repository variable
  # is configured differently.
  #
  # The outputs for this job have the same names as the workflow outputs in sub-find-cached-disks.yml
  get-available-disks:
    name: Check if cached state disks exist for ${{ inputs.network || vars.ZCASH_NETWORK }}
    # Skip PRs from external repositories, let them pass, and then Mergify will check them
    if: ${{ !startsWith(github.event_name, 'pull') || !github.event.pull_request.head.repo.fork }}
    uses: ./.github/workflows/sub-find-cached-disks.yml
    with:
      network: ${{ inputs.network || vars.ZCASH_NETWORK }}

  # Check if the cached state disks used by the tests are available for testnet.
  #
  # The outputs for this job have the same names as the workflow outputs in sub-find-cached-disks.yml
  # Some outputs are ignored, because we don't run those jobs on testnet.
  get-available-disks-testnet:
    name: Check if cached state disks exist for testnet
    if: ${{ !startsWith(github.event_name, 'pull') || !github.event.pull_request.head.repo.fork }}
    uses: ./.github/workflows/sub-find-cached-disks.yml
    with:
      network: 'Testnet'

  # Build the docker image used by the tests.
  #
  # The default network in the Zebra config in the image is mainnet, unless a manually triggered
  # workflow or repository variable is configured differently. Testnet jobs change that config to
  # testnet when running the image.
  build:
    name: Build CI Docker
    if: ${{ !startsWith(github.event_name, 'pull') || !github.event.pull_request.head.repo.fork }}
    uses: ./.github/workflows/sub-build-docker-image.yml
    with:
      dockerfile_path: ./docker/Dockerfile
      dockerfile_target: tests
      image_name: ${{ vars.CI_IMAGE_NAME }}
      no_cache: ${{ inputs.no_cache || false }}
      rust_backtrace: full
      rust_lib_backtrace: full
      rust_log: info

  # zebrad cached checkpoint state tests

  # Regenerate mandatory checkpoint Zebra cached state disks.
  #
  # Runs:
  # - on every PR update, but only if there's no available disk matching the actual state version from constants.rs
  # - on request, using workflow_dispatch with regenerate-disks
  #
  # Note: the output from get-available-disks should match with the caller workflow inputs
  regenerate-stateful-disks:
    name: Zebra checkpoint
    needs: [ build, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || github.event.inputs.regenerate-disks == 'true' }}
    with:
      app_name: zebrad
      test_id: sync-to-checkpoint
      test_description: Test sync up to mandatory checkpoint
      test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_DISK_REBUILD=1 -e ZEBRA_FORCE_USE_COLOR=1'
      needs_zebra_state: false
      saves_to_disk: true
      force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
      disk_suffix: checkpoint
      height_grep_text: 'flushing database to disk .*height.*=.*Height.*\('
    secrets: inherit
    # We want to prevent multiple checkpoint syncs running at the same time,
    # but we don't want to cancel running syncs on `main` if a new PR gets merged,
    # because we might never get a finished sync.
    #
    # See the concurrency comment on the zebrad test-full-sync job for details.
    concurrency:
      group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.regenerate-disks == 'true') }}-regenerate-stateful-disks
      cancel-in-progress: false

  # Test that Zebra syncs and fully validates a few thousand blocks from a cached mandatory checkpoint disk
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  test-stateful-sync:
    name: Zebra checkpoint update
    needs: [ regenerate-stateful-disks, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || needs.regenerate-stateful-disks.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: sync-past-checkpoint
      test_description: Test full validation sync from a cached state
      test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_CHECKPOINT_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1'
      needs_zebra_state: true
      saves_to_disk: false
      disk_suffix: checkpoint
    secrets: inherit

  # zebrad cached tip state tests

  # Test that Zebra can run a full sync on mainnet,
  # and regenerate chain tip Zebra cached state disks.
  #
  # This test always runs on mainnet.
  #
  # Runs:
  # - on schedule, as defined at the top of the workflow
  # - on every PR update, but only if the state version in constants.rs has no cached disk
  # - in manual workflow runs, when run-full-sync is 'true' and network is 'Mainnet'
  #
  # Note: the output from get-available-disks should match with the caller workflow inputs
  test-full-sync:
    name: Zebra tip
    needs: [ build, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet') }}
    with:
      app_name: zebrad
      test_id: full-sync
      test_description: Test a full sync up to the tip
      # The value of FULL_SYNC_MAINNET_TIMEOUT_MINUTES is currently ignored.
      # TODO: update the test to use {{ input.network }} instead?
      test_variables: '-e NETWORK=Mainnet -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1'
      # This test runs for longer than 6 hours, so it needs multiple jobs
      is_long_test: true
      needs_zebra_state: false
      saves_to_disk: true
      force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
      disk_suffix: tip
      height_grep_text: 'current_height.*=.*Height.*\('
    secrets: inherit
    # We want to prevent multiple full zebrad syncs running at the same time,
    # but we don't want to cancel running syncs on `main` if a new PR gets merged,
    # because we might never get a finished sync.
    #
    # Instead, we let the first sync complete, then queue the latest pending sync, cancelling any syncs in between.
    # (As the general workflow concurrency group just gets matched in Pull Requests,
    # it has no impact on this job.)
    #
    # TODO:
    # - allow multiple manual syncs on a branch by adding '-${{ github.run_id }}' when github.event.inputs.run-full-sync is true
    concurrency:
      group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-full-sync == 'true') }}-test-full-sync
      cancel-in-progress: false

  # Test that Zebra can sync to the chain tip, using a cached Zebra tip state,
  # without launching `lightwalletd`.
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  test-update-sync:
    name: Zebra tip update
    needs: [ test-full-sync, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: update-to-tip
      test_description: Test syncing to tip with a Zebra tip state
      test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_UPDATE_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
      needs_zebra_state: true
      # update the disk on every PR, to increase CI speed
      saves_to_disk: true
      force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      height_grep_text: 'current_height.*=.*Height.*\('
    secrets: inherit

  # zebra mainnet checkpoint generation tests

  # Test that Zebra can generate mainnet checkpoints after syncing to the chain tip,
  # using a cached Zebra tip state,
  #
  # This test always runs on mainnet.
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  checkpoints-mainnet:
    name: Generate checkpoints mainnet
    needs: [ test-full-sync, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: checkpoints-mainnet
      test_description: Generate Zebra checkpoints on mainnet
      # TODO: update the test to use {{ input.network }} instead?
      test_variables: '-e NETWORK=Mainnet -e GENERATE_CHECKPOINTS_MAINNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
      needs_zebra_state: true
      # test-update-sync updates the disk on every PR, so we don't need to do it here
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      height_grep_text: 'current_height.*=.*Height.*\('
    secrets: inherit

  # zebra testnet checkpoint generation tests
  #
  # These tests will fail when testnet is unstable, they should not be required to merge.
  #
  # TODO: ignore failures on testnet, so any failures don't appear in the GitHub interface.

  # Test that Zebra can run a full testnet sync, and regenerate chain tip Zebra cached state disks.
  # This job always runs on testnet, regardless of any inputs or variable settings.
  #
  # Runs:
  # - on schedule, as defined at the top of the workflow
  # - on every PR update, but only if the state version in constants.rs has no cached disk
  # - in manual workflow runs, when run-full-sync is 'true' and network is 'Testnet'
  #
  # Note: the output from get-available-disks-testnet should match with the caller workflow inputs
  test-full-sync-testnet:
    name: Zebra tip on testnet
    needs: [ build, get-available-disks-testnet ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ (github.event_name == 'schedule' && vars.SCHEDULE_TESTNET_FULL_SYNC == 'true') || !fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Testnet') }}
    with:
      app_name: zebrad
      test_id: full-sync-testnet
      test_description: Test a full sync up to the tip on testnet
      # The value of FULL_SYNC_TESTNET_TIMEOUT_MINUTES is currently ignored.
      test_variables: '-e NETWORK=Testnet -e FULL_SYNC_TESTNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1'
      network: "Testnet"
      # A full testnet sync could take 2-10 hours in April 2023.
      # The time varies a lot due to the small number of nodes.
      is_long_test: true
      needs_zebra_state: false
      saves_to_disk: true
      force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
      disk_suffix: tip
      height_grep_text: 'current_height.*=.*Height.*\('
    secrets: inherit
    # We want to prevent multiple full zebrad syncs running at the same time,
    # but we don't want to cancel running syncs on `main` if a new PR gets merged,
    # because we might never get a finished sync.
    #
    # Instead, we let the first sync complete, then queue the latest pending sync, cancelling any syncs in between.
    # (As the general workflow concurrency group just gets matched in Pull Requests,
    # it has no impact on this job.)
    #
    # TODO:
    # - allow multiple manual syncs on a branch by adding '-${{ github.run_id }}' when github.event.inputs.run-full-sync is true
    concurrency:
      group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-full-sync == 'true') }}-test-full-sync-testnet
      cancel-in-progress: false

  # Test that Zebra can generate testnet checkpoints after syncing to the chain tip,
  # using a cached Zebra tip state.
  #
  # This test always runs on testnet.
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  checkpoints-testnet:
    name: Generate checkpoints testnet
    needs: [ test-full-sync-testnet, get-available-disks-testnet ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || needs.test-full-sync-testnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: checkpoints-testnet
      test_description: Generate Zebra checkpoints on testnet
      test_variables: '-e NETWORK=Testnet -e GENERATE_CHECKPOINTS_TESTNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
      network: "Testnet"
      needs_zebra_state: true
      # update the disk on every PR, to increase CI speed
      # we don't have a test-update-sync-testnet job, so we need to update the disk here
      saves_to_disk: true
      force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      height_grep_text: 'zebra_tip_height.*=.*Height.*\('
    secrets: inherit

  # lightwalletd cached tip state tests

  # Test full sync of lightwalletd with a Zebra tip state
  #
  # Runs:
  # - on schedule, as defined at the top of the workflow
  # - on every PR update, but only if the state version in constants.rs has no cached disk
  # - in manual workflow runs, when run-lwd-sync is 'true' and network is 'Mainnet' (the network is required by the test-full-sync job)
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  lightwalletd-full-sync:
    name: lightwalletd tip
    needs: [ test-full-sync, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    # Currently the lightwalletd tests only work on Mainnet
    if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && (github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || github.event.inputs.run-lwd-sync == 'true' ) }}
    with:
      app_name: lightwalletd
      test_id: lwd-full-sync
      test_description: Test lightwalletd full sync
      test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_FULL_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
      # This test runs for longer than 6 hours, so it needs multiple jobs
      is_long_test: true
      needs_zebra_state: true
      needs_lwd_state: false
      saves_to_disk: true
      force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
      disk_prefix: lwd-cache
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      lwd_state_dir: 'lwd-cache'
      height_grep_text: 'Waiting for block: '
    secrets: inherit
    # We want to prevent multiple lightwalletd full syncs running at the same time,
    # but we don't want to cancel running syncs on `main` if a new PR gets merged,
    # because we might never get a finished sync.
    #
    # See the concurrency comment on the zebrad test-full-sync job for details.
    concurrency:
      group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-lwd-sync == 'true') }}-lightwalletd-full-sync
      cancel-in-progress: false

  # Test update sync of lightwalletd with a lightwalletd and Zebra tip state
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  lightwalletd-update-sync:
    name: lightwalletd tip update
    needs: [ lightwalletd-full-sync, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: lightwalletd
      test_id: lwd-update-sync
      test_description: Test lightwalletd update sync with both states
      test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
      needs_zebra_state: true
      needs_lwd_state: true
      saves_to_disk: true
      force_save_to_disk: ${{ inputs.force_save_to_disk || false }}
      disk_prefix: lwd-cache
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      lwd_state_dir: 'lwd-cache'
      height_grep_text: 'Waiting for block: '
    secrets: inherit

  # Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  #
  # TODO: move this job below the rest of the mainnet jobs that just use Zebra cached state
  lightwalletd-rpc-test:
    name: Zebra tip JSON-RPC
    needs: [ test-full-sync, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: lightwalletd
      test_id: fully-synced-rpc
      test_description: Test lightwalletd RPC with a Zebra tip state
      test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_RPC_CALL=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
      needs_zebra_state: true
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
    secrets: inherit

  # Test that Zebra can handle a lightwalletd send transaction RPC call, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  lightwalletd-transactions-test:
    name: lightwalletd tip send
    needs: [ lightwalletd-full-sync, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: lightwalletd
      test_id: lwd-send-transactions
      test_description: Test sending transactions via lightwalletd
      test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_TRANSACTIONS=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
      needs_zebra_state: true
      needs_lwd_state: true
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      lwd_state_dir: 'lwd-cache'
    secrets: inherit

  # Test that Zebra can handle gRPC wallet calls, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  lightwalletd-grpc-test:
    name: lightwalletd GRPC tests
    needs: [ lightwalletd-full-sync, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: lightwalletd
      test_id: lwd-grpc-wallet
      test_description: Test gRPC calls via lightwalletd
      test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_GRPC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
      needs_zebra_state: true
      needs_lwd_state: true
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      lwd_state_dir: 'lwd-cache'
    secrets: inherit

  ## getblocktemplate-rpcs using cached Zebra state on mainnet
  #
  # TODO: move these below the rest of the mainnet jobs that just use Zebra cached state

  # Test that Zebra can handle a getblocktemplate RPC call, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  get-block-template-test:
    name: get block template
    needs: [ test-full-sync, get-available-disks ]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: get-block-template
      test_description: Test getblocktemplate RPC method via Zebra's rpc server
      test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_GET_BLOCK_TEMPLATE=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
      needs_zebra_state: true
      needs_lwd_state: false
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
    secrets: inherit

  # Test that Zebra can handle a submit block RPC call, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  submit-block-test:
    name: submit block
    needs: [test-full-sync, get-available-disks]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: submit-block
      test_description: Test submitting blocks via Zebra's rpc server
      test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SUBMIT_BLOCK=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
      needs_zebra_state: true
      needs_lwd_state: false
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: "/var/cache"
      zebra_state_dir: "zebrad-cache"
    secrets: inherit

  # Test that the scanner can continue scanning where it was left when zebrad restarts.
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  scan-start-where-left-test:
    name: Scan starts where left
    needs: [test-full-sync, get-available-disks]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: scan-start-where-left
      test_description: Test that the scanner can continue scanning where it was left when zebrad restarts.
      test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
      needs_zebra_state: true
      needs_lwd_state: false
      saves_to_disk: true
      disk_suffix: tip
      root_state_path: "/var/cache"
      zebra_state_dir: "zebrad-cache"
    secrets: inherit

  # Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running.
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  scan-task-commands-test:
    name: scan task commands
    needs: [test-full-sync, get-available-disks]
    uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml
    if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: scan-task-commands
      test_description: Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running.
      test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_TASK_COMMANDS=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache"
      needs_zebra_state: true
      needs_lwd_state: false
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: "/var/cache"
      zebra_state_dir: "zebrad-cache"
    secrets: inherit

  failure-issue:
    name: Open or update issues for main branch failures
    # When a new test is added to this workflow, add it to this list.
    #
    # This list is for reliable tests that are run on the `main` branch.
    # Testnet jobs are not in this list, because we expect testnet to fail occasionally.
    needs:
      [
        regenerate-stateful-disks,
        test-full-sync,
        lightwalletd-full-sync,
        test-stateful-sync,
        test-update-sync,
        checkpoints-mainnet,
        lightwalletd-update-sync,
        lightwalletd-rpc-test,
        lightwalletd-transactions-test,
        lightwalletd-grpc-test,
        get-block-template-test,
        submit-block-test,
        scan-start-where-left-test,
        scan-task-commands-test
      ]
    # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges.
    # (PR statuses are already reported in the PR jobs list, and checked by Mergify.)
    if: (failure() && github.event.pull_request == null) || (cancelled() && github.event.pull_request == null)
    runs-on: ubuntu-latest
    steps:
      - uses: jayqi/failed-build-issue-action@v1
        with:
          title-template: "{{refname}} branch CI failed: {{eventName}} in {{workflow}}"
          # New failures open an issue with this label.
          label-name: S-ci-fail-main-branch-auto-issue
          # If there is already an open issue with this label, any failures become comments on that issue.
          always-create-new-issue: false
          github-token: ${{ secrets.GITHUB_TOKEN }}