diff --git a/.github/workflows/gcp-test-deploy.yml b/.github/workflows/gcp-test-deploy.yml new file mode 100644 index 000000000..8932cd946 --- /dev/null +++ b/.github/workflows/gcp-test-deploy.yml @@ -0,0 +1,256 @@ +name: Test instance deployment + +on: + workflow_call: + inputs: + network: + required: false + type: string + app_name: + required: false + type: string + default: 'zebra' + test_id: + required: true + type: string + test_description: + required: true + type: string + test_variables: + required: true + type: string + zebra_state_path: + required: false + type: string + default: '/zebrad-cache' + disk_prefix: + required: false + type: string + default: 'zebrad-cache' + disk_suffix: + required: false + type: string + needs_zebra_state: + required: true + type: boolean + default: false + saves_to_disk: + required: true + type: boolean + height_grep_text: + required: false + type: string + +env: + NETWORK: Mainnet + IMAGE_NAME: zebrad-test + GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra + ZONE: us-central1-a + MACHINE_TYPE: c2d-standard-16 + +jobs: + test-without-cached-state: + name: Run ${{ inputs.test_id }} test + if: ${{ !inputs.needs_zebra_state }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.7.1 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + - name: Create GCP compute instance + id: create-instance + run: | + gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ + --boot-disk-size 100GB \ + --boot-disk-type pd-ssd \ + --create-disk name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=100GB,type=pd-ssd \ + --container-image debian:buster \ + --container-restart-policy=never \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --scopes cloud-platform \ + --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ + --tags ${{ inputs.app_name }} \ + --zone ${{ env.ZONE }} + sleep 60 + + - name: Run ${{ inputs.test_id }} test + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + sudo mkfs.ext4 /dev/sdb \ + && \ + docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ + ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ + && \ + docker run ${{ inputs.test_variables }} -t --name ${{ inputs.test_id }} \ + --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.zebra_state_path }} \ + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}" + + - name: Get state version from constants.rs + run: | + LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" $GITHUB_WORKSPACE/zebra-state/src/constants.rs | grep -oE "[0-9]+" | tail -n1) + echo "STATE_VERSION: $LOCAL_STATE_VERSION" + + echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV + + - name: Get sync height from logs + run: | + SYNC_HEIGHT="" + + DOCKER_LOGS=$(\ + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command="docker logs ${{ inputs.test_id }} --tail 20") + + SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE '${{ inputs.height_grep_text }}\([0-9]+\)' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]]) + echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV + + # Create image from disk that will be used for following tests + # Force the image creation as the disk is still attached even though is not being used by the container + - name: Create image from state disk + if: ${{ inputs.saves_to_disk }} + run: | + gcloud compute images create ${{ inputs.disk_prefix }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-${{ inputs.disk_suffix }} \ + --force \ + --source-disk=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ + --source-disk-zone=${{ env.ZONE }} \ + --storage-location=us \ + --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}" + + - name: Delete test instance + # If the disk generation step timeouts (+6 hours) the previous step (creating the image) willl be skipped. + # Even if the instance continues running, no image will be created, so it's better to delete it. + if: always() + continue-on-error: true + run: | + INSTANCE=$(gcloud compute instances list --filter=${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)') + if [ -z "${INSTANCE}" ]; then + echo "No instance to delete" + else + gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet + fi + + test-with-cached-state: + name: Run ${{ inputs.test_id }} test + if: ${{ inputs.needs_zebra_state }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.7.1 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Before executing any further steps, validate the local state and remote version are the same, + # or at least that the local state version is greater than the available cached state version from main. + # + # Aftwards, get the disk name to be used on further steps + - name: Validate local state version with cached state + id: get-disk-name + run: | + LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1) + echo "LOCAL_STATE_VERSION: $LOCAL_STATE_VERSION" + + ZEBRA_STATE_DISK=$(gcloud compute images list --filter="name~${{ inputs.disk_prefix }} AND name~-${{ inputs.disk_suffix }}" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) + echo "Disk: $ZEBRA_STATE_DISK" + echo "Description: $(gcloud compute images describe $ZEBRA_STATE_DISK --format='value(DESCRIPTION)')" + + GCP_STATE_VERSION=$(echo "$ZEBRA_STATE_DISK" | grep -oE "v[0-9]+" | grep -oE "[0-9]+") + echo "GCP_STATE_VERSION: $GCP_STATE_VERSION" + + if [[ "$LOCAL_STATE_VERSION" -lt "$GCP_STATE_VERSION" ]]; then echo "Local version is lower than cached version" && exit 1; fi + + echo "ZEBRA_CACHED_DISK_NAME=$ZEBRA_STATE_DISK" >> $GITHUB_ENV + + # Creates Compute Engine virtual machine instance w/ disks + - name: Create GCP compute instance + id: create-instance + run: | + gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ + --boot-disk-size 100GB \ + --boot-disk-type pd-ssd \ + --create-disk image=${{ env.ZEBRA_CACHED_DISK_NAME }},name="${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=100GB,type=pd-ssd \ + --container-image debian:buster \ + --container-restart-policy=never \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --scopes cloud-platform \ + --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ + --tags ${{ inputs.app_name }} \ + --zone ${{ env.ZONE }} + sleep 60 + + - name: Run ${{ inputs.test_id }} test + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ + ${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ + && \ + docker run ${{ inputs.test_variables }} -t --name ${{ inputs.test_id }} \ + --mount type=volume,src=${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.zebra_state_path }} \ + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}" + + - name: Delete test instance + # We don't want to leave a failed instance in GCP using resources + if: always() + continue-on-error: true + run: | + INSTANCE=$(gcloud compute instances list --filter=${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)') + if [ -z "${INSTANCE}" ]; then + echo "No instance to delete" + else + gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet + fi diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5dc2835c0..2f1544aac 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -107,3 +107,6 @@ jobs: steps: - uses: actions/checkout@v3.0.2 - uses: reviewdog/action-actionlint@v1.24.0 + with: + reporter: github-check + fail_on_error: false diff --git a/.github/workflows/test-full-sync.yml b/.github/workflows/test-full-sync.yml deleted file mode 100644 index 33a2c2d06..000000000 --- a/.github/workflows/test-full-sync.yml +++ /dev/null @@ -1,194 +0,0 @@ -name: Full sync test - -on: - workflow_dispatch: - inputs: - network: - default: 'Mainnet' - description: 'Network to deploy: Mainnet or Testnet' - required: true - checkpoint_sync: - default: 'true' - description: 'Configures `zebrad` to use as many checkpoints as possible' - required: true - pull_request: - branches: - - main - paths: - # code and tests (including full sync acceptance test changes) - # TODO: ignore changes in test code that isn't used in the full sync test - - '**/*.rs' - # hard-coded checkpoints - # TODO: ignore changes to proptest seed .txt files - - '**/*.txt' - # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' - # workflow definitions - - 'docker/**' - - '.github/workflows/test-full-sync.yml' - push: - branches: - - main - paths: - # code and tests (including full sync acceptance test changes) - # TODO: ignore changes in test code that isn't used in the full sync test - - '**/*.rs' - # hard-coded checkpoints - # TODO: ignore changes to proptest seed .txt files - - '**/*.txt' - # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' - # workflow definitions - - 'docker/**' - - '.github/workflows/test-full-sync.yml' - - '.github/workflows/docker-image-build.yml' - -env: - NETWORK: Mainnet - PROJECT_ID: zealous-zebra - IMAGE_NAME: zebrad-test - GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra - REGION: us-central1 - ZONE: us-central1-a - MACHINE_TYPE: c2d-standard-16 - -jobs: - build: - # TODO add `startsWith(github.head_ref, 'mergify/merge-queue/')` to the condition to - # only run on Mergify head branches, and on manual dispatch: - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1 - if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - uses: ./.github/workflows/docker-image-build.yml - with: - dockerfile_path: ./docker/Dockerfile - dockerfile_target: tester - image_name: zebrad-test - network: Mainnet - checkpoint_sync: true - rust_backtrace: full - rust_lib_backtrace: full - colorbt_show_hidden: '1' - zebra_skip_ipv6_tests: '1' - rust_log: info - - # Test that Zebra can run a full mainnet sync after a PR is approved - test-full-sync: - name: Test full Mainnet sync - runs-on: ubuntu-latest - needs: [build] - permissions: - contents: 'read' - id-token: 'write' - steps: - - uses: actions/checkout@v3.0.2 - with: - persist-credentials: false - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - - name: Downcase network name for disks - run: | - NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }} - echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV - - # Setup gcloud CLI - - name: Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@v0.7.1 - with: - workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' - service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' - token_format: 'access_token' - - # Creates Compute Engine virtual machine instance w/ disks - - name: Create GCP compute instance - run: | - gcloud compute instances create-with-container "full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ - --boot-disk-size 100GB \ - --boot-disk-type pd-ssd \ - --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip",device-name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip",size=100GB,type=pd-ssd \ - --container-mount-disk mount-path="/zebrad-cache",name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip" \ - --container-image debian:buster \ - --container-restart-policy=never \ - --machine-type ${{ env.MACHINE_TYPE }} \ - --scopes cloud-platform \ - --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ - --tags zebrad \ - --zone "${{ env.ZONE }}" - sleep 60 - - - name: Full sync - id: full-sync - run: | - gcloud compute ssh \ - full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command \ - "docker run -e TEST_FULL_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 -t --name full-sync \ - --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip,target=/zebrad-cache \ - ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}" - - EXIT_CODE=$(\ - gcloud compute ssh \ - full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command="docker wait full-sync") - - exit ${EXIT_CODE} - - - name: Get state version from constants.rs - run: | - STATE_VERSION="" - - LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" $GITHUB_WORKSPACE/zebra-state/src/constants.rs | grep -oE "[0-9]+" | tail -n1) - echo "STATE_VERSION: $LOCAL_STATE_VERSION" - - echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV - - - name: Get sync height from logs - run: | - SYNC_HEIGHT="" - - DOCKER_LOGS=$(\ - gcloud compute ssh \ - full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command="docker logs full-sync --tail 20") - - SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE 'finished initial sync to chain tip, using gossiped blocks sync_percent=100.000 % current_height=Height\([0-9]+\)' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]]) - echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV - - # Create image from disk - # Force the image creation as the disk is still attached, even though is not being used by the container - - name: Create image from state disk - run: | - gcloud compute images create zebrad-cache-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-tip \ - --force \ - --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-tip \ - --source-disk-zone=${{ env.ZONE }} \ - --storage-location=us \ - --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}" - - - name: Delete test instance - # If the `full-sync` step timeouts (+6 hours) the previous step (creating the image) willl be skipped. - # Even if the instance continues running, no image will be created, so it's better to delete it. - if: always() - continue-on-error: true - run: | - INSTANCE=$(gcloud compute instances list --filter=full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)') - if [ -z "${INSTANCE}" ]; then - echo "No instance to delete" - else - gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet - fi diff --git a/.github/workflows/test.patch.yml b/.github/workflows/test.patch.yml index 4b532f99a..9f2199c3f 100644 --- a/.github/workflows/test.patch.yml +++ b/.github/workflows/test.patch.yml @@ -5,13 +5,38 @@ on: branches: - main paths-ignore: + # code and tests - '**/*.rs' + # hard-coded checkpoints and proptest regressions - '**/*.txt' + # test data snapshots - '**/*.snap' + # dependencies - '**/Cargo.toml' - '**/Cargo.lock' + # workflow definitions - 'docker/**' - '.github/workflows/test.yml' + - '.github/workflows/gcp-test-deploy.yml' + - '.github/workflows/docker-image-build.yml' + + push: + branches: + - main + paths-ignore: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # test data snapshots + - '**/*.snap' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # workflow definitions + - 'docker/**' + - '.github/workflows/test.yml' + - '.github/workflows/gcp-test-deploy.yml' - '.github/workflows/docker-image-build.yml' jobs: @@ -65,3 +90,11 @@ jobs: needs: [build, regenerate-stateful-disks] steps: - run: 'echo "No build required"' + + # Test that Zebra can run a full mainnet sync after a PR is approved + test-full-sync: + name: Test a full sync up to the tip + runs-on: ubuntu-latest + needs: [build, regenerate-stateful-disks] + steps: + - run: 'echo "No build required"' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92adbc26b..e3c2ab85c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,12 @@ on: default: false description: 'Just update stateful disks' required: true + run-full-sync: + type: boolean + default: false + description: 'Just trigger a full sync' + required: true + pull_request: branches: - main @@ -32,6 +38,8 @@ on: # workflow definitions - 'docker/**' - '.github/workflows/test.yml' + - '.github/workflows/gcp-test-deploy.yml' + - '.github/workflows/docker-image-build.yml' push: branches: @@ -49,24 +57,37 @@ on: # workflow definitions - 'docker/**' - '.github/workflows/test.yml' + - '.github/workflows/gcp-test-deploy.yml' - '.github/workflows/docker-image-build.yml' env: - NETWORK: Mainnet - PROJECT_ID: zealous-zebra + # TODO: use the output from ./.github/workflows/docker-image-build.yml IMAGE_NAME: zebrad-test GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra - REGION: us-central1 - ZONE: us-central1-a - MACHINE_TYPE: c2d-standard-4 jobs: + changed-files: + runs-on: ubuntu-latest + name: Test changed-files + outputs: + modified_files: ${{ steps.changed-files.outputs.modified_files }} + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v19 + build: uses: ./.github/workflows/docker-image-build.yml with: dockerfile_path: ./docker/Dockerfile dockerfile_target: tester image_name: zebrad-test + # TODO: validate how to use variable/conditional values for Testnet network: Mainnet checkpoint_sync: true rust_backtrace: full @@ -132,6 +153,7 @@ jobs: docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} docker run --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored + # Test launching lightwalletd with an empty lightwalletd and Zebra state test-lightwalletd-integration: name: Test integration with lightwalletd runs-on: ubuntu-latest @@ -143,245 +165,53 @@ jobs: with: short-length: 7 - - name: Run tests with included lightwalletd binary + - name: Run tests with empty lightwalletd launch run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e ZEBRA_TEST_LIGHTWALLETD --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance -- lightwalletd_integration --nocapture + docker run -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance -- lightwalletd_integration --nocapture env: ZEBRA_TEST_LIGHTWALLETD: '1' regenerate-stateful-disks: - name: Regenerate stateful disks - runs-on: ubuntu-latest - needs: build - outputs: - any_changed: ${{ steps.changed-files-specific.outputs.any_changed }} - permissions: - contents: 'read' - id-token: 'write' - steps: - - uses: actions/checkout@v3.0.2 - with: - persist-credentials: false - fetch-depth: '2' - - # TODO move the `changed-files-specific` step to the build job for a better dependency tree - # Only run this job if the database format version has (likely) changed. - # - # If we have accidentally changed the format, but not changed the version, - # we want to run with the old cached state, so this job fails. - # - # If we change the state path without changing the version, - # this job will take a few hours, because it will do a full rebuild. - - name: Get specific changed files - id: changed-files-specific - uses: tj-actions/changed-files@v19 - with: - files: | - zebra-state/**/constants.rs - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - - name: Downcase network name for disks - run: | - NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }} - echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV - - # Setup gcloud CLI - - name: Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@v0.7.1 - with: - workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' - service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' - token_format: 'access_token' - - - name: Create GCP compute instance - id: create-instance - if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' || github.event_name == 'push'}} - run: | - gcloud compute instances create-with-container "sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ - --boot-disk-size 100GB \ - --boot-disk-type pd-ssd \ - --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint",device-name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint",size=100GB,type=pd-ssd \ - --container-mount-disk mount-path="/zebrad-cache",name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint" \ - --container-image debian:buster \ - --container-restart-policy=never \ - --machine-type ${{ env.MACHINE_TYPE }} \ - --scopes cloud-platform \ - --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ - --tags zebrad \ - --zone "${{ env.ZONE }}" - sleep 60 - - - name: Regenerate stateful disks - id: sync-to-checkpoint - if: ${{ steps.create-instance.outcome == 'success' }} - run: | - gcloud compute ssh \ - sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command \ - "docker run -t --name sync-to-checkpoint \ - --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint,target=/zebrad-cache \ - ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \ - cargo test --locked --release --features enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ env.NETWORK }}" - - EXIT_CODE=$(\ - gcloud compute ssh \ - sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command="docker wait sync-to-checkpoint") - - exit ${EXIT_CODE} - - - name: Get state version from constants.rs - if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} - run: | - STATE_VERSION="" - - LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" $GITHUB_WORKSPACE/zebra-state/src/constants.rs | grep -oE "[0-9]+" | tail -n1) - echo "STATE_VERSION: $LOCAL_STATE_VERSION" - - echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV - - - name: Get sync height from logs - if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} - run: | - SYNC_HEIGHT="" - - DOCKER_LOGS=$(\ - gcloud compute ssh \ - sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command="docker logs sync-to-checkpoint --tail 20") - - SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE 'flushing database to disk height=Height\([0-9]+\)' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]]) - - echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV - - # Create image from disk that will be used for the sync past mandatory checkpoint test - # Force the image creation as the disk is still attached even though is not being used by the container - - name: Create image from state disk - # Only run if the earlier step succeeds - if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} - run: | - gcloud compute images create zebrad-cache-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-checkpoint \ - --force \ - --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.NETWORK }}-checkpoint \ - --source-disk-zone=${{ env.ZONE }} \ - --storage-location=us \ - --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}" - - - name: Delete test instance - # If the `sync-to-checkpoint` step timeouts (+6 hours) the previous step (creating the image) willl be skipped. - # Even if the instance continues running, no image will be created, so it's better to delete it. - if: always() - continue-on-error: true - run: | - INSTANCE=$(gcloud compute instances list --filter=sync-to-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)') - if [ -z "${INSTANCE}" ]; then - echo "No instance to delete" - else - gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet - fi + needs: [ build, changed-files] + uses: ./.github/workflows/gcp-test-deploy.yml + # Only run this job if the database format version has (likely) changed. + if: ${{ (contains(needs.changed-files.outputs.modified_files, 'zebra-state/src/constants.rs') && github.event_name == 'push') || github.event.inputs.regenerate-disks == 'true' }} + with: + test_id: sync-to-checkpoint + test_description: Test sync up to mandatory checkpoint + test_variables: '-e TEST_DISK_REBUILD=1 -e ZEBRA_FORCE_USE_COLOR=1' + needs_zebra_state: false + saves_to_disk: true + disk_suffix: checkpoint + height_grep_text: 'flushing database to disk height=Height' # Test that Zebra syncs and fully validates a few thousand blocks from a cached post-checkpoint state test-stateful-sync: - name: Test full validation sync from cached state - runs-on: ubuntu-latest - needs: [build, regenerate-stateful-disks] - permissions: - contents: 'read' - id-token: 'write' - steps: - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 + needs: build + uses: ./.github/workflows/gcp-test-deploy.yml + if: ${{ github.event.inputs.regenerate-disks != 'true' }} + with: + test_id: sync-past-checkpoint + test_description: Test full validation sync from a cached state + test_variables: '-e TEST_CHECKPOINT_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1' + needs_zebra_state: true + saves_to_disk: false + disk_suffix: checkpoint - - name: Downcase network name for disks - run: | - NETWORK_CAPS=${{ github.event.inputs.network || env.NETWORK }} - echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV - - # Setup gcloud CLI - - name: Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@v0.7.1 - with: - workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' - service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' - token_format: 'access_token' - - - name: Get disk state name from gcloud - id: get-disk-name - run: | - GCP_STATE_DISK=$(gcloud compute images list --filter="name~zebrad-cache AND name~-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) - echo "Disk: $GCP_STATE_DISK" - echo "Description: $(gcloud compute images describe $GCP_STATE_DISK --format='value(DESCRIPTION)')" - - echo "CACHED_DISK_NAME=$GCP_STATE_DISK" >> $GITHUB_ENV - - # Creates Compute Engine virtual machine instance w/ disks - - name: Create GCP compute instance - id: create-instance - run: | - gcloud compute instances create-with-container "sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ - --boot-disk-size 100GB \ - --boot-disk-type pd-ssd \ - --create-disk image=${{ env.CACHED_DISK_NAME }},name="${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }}",size=100GB,type=pd-ssd \ - --container-mount-disk mount-path="/zebrad-cache",name="${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }}" \ - --container-image debian:buster \ - --container-restart-policy=never \ - --machine-type ${{ env.MACHINE_TYPE }} \ - --scopes cloud-platform \ - --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ - --tags zebrad \ - --zone "${{ env.ZONE }}" - sleep 60 - - - name: Sync past mandatory checkpoint - id: sync-past-checkpoint - run: | - gcloud compute ssh \ - sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command \ - "docker run -t --name sync-past-checkpoint \ - --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/${{ env.CACHED_DISK_NAME }}-${{ env.GITHUB_SHA_SHORT }},target=/zebrad-cache \ - ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \ - cargo test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ env.NETWORK }}" - - EXIT_CODE=$(\ - gcloud compute ssh \ - sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command="docker wait sync-past-checkpoint") - - exit ${EXIT_CODE} - - - name: Delete test instance - # We don't want to leave a failed instance in GCP using resources - if: always() - continue-on-error: true - run: | - INSTANCE=$(gcloud compute instances list --filter=sync-past-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)') - if [ -z "${INSTANCE}" ]; then - echo "No instance to delete" - else - gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet - fi + # Test that Zebra can run a full mainnet sync after a PR is approved + test-full-sync: + needs: build + uses: ./.github/workflows/gcp-test-deploy.yml + # TODO change `github.ref_name == 'main'` to `startsWith(github.head_ref, 'mergify/merge-queue/')` + # to only run on Mergify head branches, and on manual dispatch: + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1 + if: ${{ (github.event_name == 'push' && github.ref_name == 'main') || github.event.inputs.run-full-sync == 'true' }} + with: + test_id: full-sync-to-tip + test_description: Test a full sync up to the tip + test_variables: '-e TEST_FULL_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600' + needs_zebra_state: false + saves_to_disk: true + disk_suffix: tip + height_grep_text: 'finished initial sync to chain tip, using gossiped blocks sync_percent=100.* current_height=Height' diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 11ac3cf90..162ca00b6 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -23,6 +23,10 @@ case "$1" in exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--workspace" "--" "--include-ignored" elif [[ "$TEST_FULL_SYNC" -eq "1" ]]; then exec cargo "test" "--locked" "--release" "--features" "enable-sentry" "--test" "acceptance" "--" "--nocapture" "--ignored" "full_sync_mainnet" + elif [[ "$TEST_DISK_REBUILD" -eq "1" ]]; then + exec cargo "test" "--locked" "--release" "--features" "enable-sentry,test_sync_to_mandatory_checkpoint_${NETWORK,,}" "--manifest-path" "zebrad/Cargo.toml" "sync_to_mandatory_checkpoint_${NETWORK,,}" + elif [[ "$TEST_CHECKPOINT_SYNC" -eq "1" ]]; then + exec cargo "test" "--locked" "--release" "--features" "enable-sentry,test_sync_past_mandatory_checkpoint_${NETWORK,,}" "--manifest-path" "zebrad/Cargo.toml" "sync_past_mandatory_checkpoint_${NETWORK,,}" else exec "$@" fi