2022-03-04 00:12:22 -08:00
name : Full sync test
on :
workflow_dispatch :
inputs :
network :
default : 'Mainnet'
2022-03-07 18:46:12 -08:00
pull_request :
2022-03-04 00:12:22 -08:00
branches :
- main
paths :
2022-03-06 17:22:46 -08:00
# code and tests (including full sync acceptance test changes)
# TODO: ignore changes in test code that isn't used in the full sync test
2022-03-04 00:12:22 -08:00
- '**/*.rs'
# hard-coded checkpoints
2022-03-06 17:22:46 -08:00
# TODO: ignore changes to proptest seed .txt files
2022-03-04 00:12:22 -08:00
- '**/*.txt'
# dependencies
- '**/Cargo.toml'
- '**/Cargo.lock'
# workflow definitions
- 'docker/**'
2022-03-04 11:55:32 -08:00
- '.github/workflows/test-full-sync.yml'
2022-03-04 00:12:22 -08:00
env :
CARGO_INCREMENTAL : '1'
ZEBRA_SKIP_IPV6_TESTS : "1"
2022-03-08 07:09:41 -08:00
RUST_BACKTRACE : full
RUST_LIB_BACKTRACE : full
COLORBT_SHOW_HIDDEN : '1'
2022-03-04 00:12:22 -08:00
NETWORK : Mainnet
2022-03-18 14:25:35 -07:00
PROJECT_ID : zealous-zebra
GAR_BASE : us-docker.pkg.dev/zealous-zebra/zebra
GCR_BASE : gcr.io/zealous-zebra
2022-03-04 00:12:22 -08:00
REGION : us-central1
ZONE : us-central1-a
MACHINE_TYPE : c2d-standard-16
IMAGE_NAME : zebrad-test
2022-03-04 11:55:32 -08:00
jobs :
build :
2022-03-09 00:00:58 -08:00
# only run on Mergify head branches, and on manual dispatch:
2022-03-07 18:46:12 -08:00
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1
2022-03-09 00:03:25 -08:00
if : startsWith(github.head_ref, 'mergify/merge-queue/') || github.event_name == 'workflow_dispatch'
2022-03-04 11:55:32 -08:00
name : Build images
timeout-minutes : 210
runs-on : ubuntu-latest
2022-03-18 14:25:35 -07:00
permissions :
contents : 'read'
id-token : 'write'
2022-03-04 11:55:32 -08:00
steps :
2022-03-08 19:38:38 -08:00
- uses : actions/checkout@v3.0.0
2022-03-04 11:55:32 -08:00
with :
persist-credentials : false
- name : Inject slug/short variables
uses : rlespinasse/github-slug-action@v4
2022-03-21 16:07:01 -07:00
with :
short-length : 7
2022-03-04 11:55:32 -08:00
2022-03-07 18:46:12 -08:00
# Automatic tag management and OCI Image Format Specification for labels
2022-03-04 11:55:32 -08:00
- name : Docker meta
id : meta
uses : docker/metadata-action@v3.6.2
with :
# list of Docker images to use as base name for tags
images : |
${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}
${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}
# generate Docker tags based on the following events/attributes
tags : |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
# Setup Docker Buildx to allow use of docker cache layers from GH
- name : Set up Docker Buildx
id : buildx
uses : docker/setup-buildx-action@v1
2022-03-18 14:25:35 -07:00
- name : Authenticate to Google Cloud
id : auth
uses : google-github-actions/auth@v0.6.0
with :
workload_identity_provider : 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
service_account : 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
token_format : 'access_token'
2022-03-04 11:55:32 -08:00
- name : Login to Google Artifact Registry
2022-03-08 01:14:10 -08:00
uses : docker/login-action@v1.14.1
2022-03-04 11:55:32 -08:00
with :
registry : us-docker.pkg.dev
2022-03-18 14:25:35 -07:00
username : oauth2accesstoken
password : ${{ steps.auth.outputs.access_token }}
2022-03-04 11:55:32 -08:00
- name : Login to Google Container Registry
2022-03-08 01:14:10 -08:00
uses : docker/login-action@v1.14.1
2022-03-04 11:55:32 -08:00
with :
registry : gcr.io
2022-03-18 14:25:35 -07:00
username : oauth2accesstoken
password : ${{ steps.auth.outputs.access_token }}
2022-03-04 11:55:32 -08:00
# Build and push image to Google Artifact Registry
- name : Build & push
id : docker_build
2022-03-15 15:35:18 -07:00
uses : docker/build-push-action@v2.10.0
2022-03-04 11:55:32 -08:00
with :
target : tester
context : .
file : ./docker/Dockerfile
tags : ${{ steps.meta.outputs.tags }}
labels : ${{ steps.meta.outputs.labels }}
build-args : |
NETWORK=${{ github.event.inputs.network || env.NETWORK }}
SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
2022-03-08 07:09:41 -08:00
RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}
RUST_LIB_BACKTRACE=${{ env.RUST_LIB_BACKTRACE }}
COLORBT_SHOW_HIDDEN=${{ env.COLORBT_SHOW_HIDDEN }}
2022-03-04 11:55:32 -08:00
ZEBRA_SKIP_NETWORK_TESTS="1"
CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
RUST_LOG=debug
SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
push : true
2022-03-09 17:12:34 -08:00
cache-from : type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:buildcache
cache-to : type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
2022-03-04 11:55:32 -08:00
2022-03-04 00:12:22 -08:00
# Test that Zebra can run a full mainnet sync after a PR is approved
test-full-sync :
name : Test full Mainnet sync
runs-on : ubuntu-latest
2022-03-04 11:55:32 -08:00
needs : [ build ]
2022-03-18 14:25:35 -07:00
permissions :
contents : 'read'
id-token : 'write'
2022-03-04 00:12:22 -08:00
steps :
- name : Inject slug/short variables
uses : rlespinasse/github-slug-action@v4
2022-03-21 16:07:01 -07:00
with :
short-length : 7
2022-03-04 00:12:22 -08:00
# Setup gcloud CLI
- name : Authenticate to Google Cloud
id : auth
2022-03-18 14:25:35 -07:00
uses : google-github-actions/auth@v0.6.0
2022-03-04 00:12:22 -08:00
with :
2022-03-18 14:25:35 -07:00
workload_identity_provider : 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
service_account : 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
token_format : 'access_token'
2022-03-04 00:12:22 -08:00
2022-03-09 13:10:05 -08:00
# Check if our destination compute instance exists and delete it
- name : Delete existing instance with same SHA
id : delete-old-instance
run : |
INSTANCE=$(gcloud compute instances list --filter=full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
if [ -z "${INSTANCE}" ]; then
echo "No instance to delete"
else
gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet
fi
2022-03-04 00:12:22 -08:00
# Creates Compute Engine virtual machine instance w/ disks
- name : Create GCP compute instance
id : create-instance
run : |
2022-03-04 11:55:32 -08:00
gcloud compute instances create-with-container "full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
2022-03-04 00:12:22 -08:00
--boot-disk-size 100GB \
2022-03-10 14:10:59 -08:00
--boot-disk-type pd-ssd \
2022-03-04 11:55:32 -08:00
--container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \
2022-03-04 00:12:22 -08:00
--container-restart-policy=never \
--container-stdin \
--container-tty \
--container-env=ZEBRA_SKIP_IPV6_TESTS=1,TEST_FULL_SYNC=1,ZEBRA_FORCE_USE_COLOR=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
--machine-type ${{ env.MACHINE_TYPE }} \
--scopes cloud-platform \
--metadata=google-monitoring-enabled=true,google-logging-enabled=true \
--tags zebrad \
--zone "${{ env.ZONE }}"
# TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
# This TODO relates to the following issues:
# https://github.com/actions/runner/issues/241
# https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
- name : Get container name from logs
id : get-container-name
2022-03-09 13:10:05 -08:00
if : ${{ steps.create-instance.outcome == 'success' }}
2022-03-04 00:12:22 -08:00
run : |
2022-03-04 11:55:32 -08:00
INSTANCE_ID=$(gcloud compute instances describe full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
2022-03-04 00:12:22 -08:00
echo "Using instance: $INSTANCE_ID"
2022-03-04 11:55:32 -08:00
while [[ ${CONTAINER_NAME} != *"full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}"* ]]; do
CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
2022-03-04 00:12:22 -08:00
echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
sleep 10
done
2022-03-04 11:55:32 -08:00
CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
2022-03-04 00:12:22 -08:00
echo "::set-output name=zebra_container::$CONTAINER_NAME"
2022-03-09 13:10:05 -08:00
sleep 60
2022-03-04 00:12:22 -08:00
- name : Full sync mainnet
id : full-sync-mainnet
2022-03-09 13:10:05 -08:00
if : ${{ steps.create-instance.outcome == 'success' }}
2022-03-04 00:12:22 -08:00
run : |
gcloud compute ssh \
2022-03-04 11:55:32 -08:00
full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
2022-03-04 00:12:22 -08:00
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
env :
ZEBRA_CONTAINER : ${{ steps.get-container-name.outputs.zebra_container }}
- name : Delete test instance
# Do not delete the instance if the sync timeouts in GitHub
if : ${{ steps.full-sync-mainnet.outcome == 'success' || steps.full-sync-mainnet.outcome == 'failure' }}
continue-on-error : true
run : |
2022-03-09 13:10:05 -08:00
gcloud compute instances delete "full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" --zone "${{ env.ZONE }}" --delete-disks all --quiet