zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

ci(ssh): revert using `ssh-compute` action & increase sshd connection limit (#5367) 

* Revert "ci(ssh): connect using `ssh-compute` action by Google (#5330)"

This reverts commit b366d6e7bb.

* ci(ssh): use sudo for docker commands if user is not root

* ci(ssh): specify the service account to connect with

* ci(ssh): increase the Google Cloud instance sshd connection limit

* chore: add a new line at the end of the script

* chore: update our VM image to bullseye

* chore: fix `tj-actions/changed-files` file comparison
This commit is contained in:
Gustavo Valverde 2022-10-10 20:11:49 -04:00 committed by GitHub
parent 3bc8f0971f
commit 658fbd923a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 341 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/continous-integration-docker.yml vendored
View File

@ -281,7 +281,6 @@ jobs:
saves_to_disk: true
disk_suffix: checkpoint
height_grep_text: 'flushing database to disk .*height.*=.*Height.*\('
secrets: inherit
# Test that Zebra syncs and fully validates a few thousand blocks from a cached mandatory checkpoint disk
#
@ -300,7 +299,6 @@ jobs:
needs_zebra_state: true
saves_to_disk: false
disk_suffix: checkpoint
secrets: inherit
# zebrad cached tip state tests
@ -330,7 +328,6 @@ jobs:
saves_to_disk: true
disk_suffix: tip
height_grep_text: 'current_height.*=.*Height.*\('
secrets: inherit
# We don't want to cancel running full syncs on `main` if a new PR gets merged,
# because we might never finish a full sync during busy weeks. Instead, we let the
# first sync complete, then queue the latest pending sync, cancelling any syncs in between.
@ -370,7 +367,6 @@ jobs:
root_state_path: '/var/cache'
zebra_state_dir: 'zebrad-cache'
height_grep_text: 'current_height.*=.*Height.*\('
secrets: inherit
# lightwalletd cached tip state tests
@ -404,7 +400,6 @@ jobs:
zebra_state_dir: 'zebrad-cache'
lwd_state_dir: 'lwd-cache'
height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )'
secrets: inherit
# Test update sync of lightwalletd with a lightwalletd and Zebra tip state
# Runs:
@ -433,7 +428,6 @@ jobs:
zebra_state_dir: 'zebrad-cache'
lwd_state_dir: 'lwd-cache'
height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )'
secrets: inherit
# Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state
#
@ -458,7 +452,6 @@ jobs:
disk_suffix: tip
root_state_path: '/var/cache'
zebra_state_dir: 'zebrad-cache'
secrets: inherit
# Test that Zebra can handle a lightwalletd send transaction RPC call, using a cached Zebra tip state
#
@ -486,7 +479,6 @@ jobs:
root_state_path: '/var/cache'
zebra_state_dir: 'zebrad-cache'
lwd_state_dir: 'lwd-cache'
secrets: inherit
# Test that Zebra can handle gRPC wallet calls, using a cached Zebra tip state
#
@ -513,4 +505,3 @@ jobs:
root_state_path: '/var/cache'
zebra_state_dir: 'zebrad-cache'
lwd_state_dir: 'lwd-cache'
secrets: inherit

572
.github/workflows/deploy-gcp-tests.yml vendored
View File

@ -79,9 +79,6 @@ on:
type: string
default: 'zebra'
description: 'Application name, used to work out when a job is an update job'
secrets:
GCP_SSH_PRIVATE_KEY:
required: true
env:
# where we get the Docker image from
@ -145,11 +142,12 @@ jobs:
--boot-disk-size 200GB \
--boot-disk-type pd-ssd \
--create-disk name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=200GB,type=pd-ssd \
--container-image debian:buster \
--container-image debian:bullseye \
--container-restart-policy=never \
--machine-type ${{ env.MACHINE_TYPE }} \
--scopes cloud-platform \
--metadata=google-monitoring-enabled=true,google-logging-enabled=true \
--metadata-from-file=startup-script=.github/workflows/scripts/gcp-vm-startup-script.sh \
--tags ${{ inputs.app_name }} \
--zone ${{ env.ZONE }}
sleep 60
@ -158,17 +156,21 @@ jobs:
#
# SSH into the just created VM, and create a docker volume with the newly created disk.
- name: Create ${{ inputs.test_id }} Docker volume
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo mkfs.ext4 -v /dev/sdb \
&& \
sudo docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \
${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo mkfs.ext4 -v /dev/sdb \
&& \
sudo docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \
${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \
"
# launch the test, if it doesn't use any cached state
launch-without-cached-state:
@ -208,20 +210,25 @@ jobs:
# Launch the test without any cached state
- name: Launch ${{ inputs.test_id }} test
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker run \
--name ${{ inputs.test_id }} \
--tty \
--detach \
${{ inputs.test_variables }} \
--mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \
${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker run \
--name ${{ inputs.test_id }} \
--tty \
--detach \
${{ inputs.test_variables }} \
--mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \
${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \
"
# set up the test, if it uses cached state
# each test runs one of the *-with/without-cached-state job series, and skips the other
@ -357,11 +364,12 @@ jobs:
--boot-disk-size 200GB \
--boot-disk-type pd-ssd \
--create-disk image=${{ env.CACHED_DISK_NAME }},name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=200GB,type=pd-ssd \
--container-image debian:buster \
--container-image debian:bullseye \
--container-restart-policy=never \
--machine-type ${{ env.MACHINE_TYPE }} \
--scopes cloud-platform \
--metadata=google-monitoring-enabled=true,google-logging-enabled=true \
--metadata-from-file=startup-script=.github/workflows/scripts/gcp-vm-startup-script.sh \
--tags ${{ inputs.app_name }} \
--zone ${{ env.ZONE }}
sleep 60
@ -373,19 +381,23 @@ jobs:
# (The cached state and disk are usually the same size,
# but the cached state can be smaller if we just increased the disk size.)
- name: Create ${{ inputs.test_id }} Docker volume
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo e2fsck -v -f -p /dev/sdb \
&& \
sudo resize2fs -p /dev/sdb \
&& \
sudo docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \
${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo e2fsck -v -f -p /dev/sdb \
&& \
sudo resize2fs -p /dev/sdb \
&& \
sudo docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \
${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \
"
# launch the test, if it uses cached state
launch-with-cached-state:
@ -427,7 +439,7 @@ jobs:
# Each test runs one of the "Launch test" steps, and skips the other.
#
# SSH into the just created VM, and create a Docker container to run the incoming test
# from ${{ inputs.test_id }}, then mount the docker volume created in the previous job.
# from ${{ inputs.test_id }}, then mount the sudo docker volume created in the previous job.
#
# The disk mounted in the VM is located at /dev/sdb, we mount the root `/` of this disk to the docker
# container in one path:
@ -445,26 +457,30 @@ jobs:
# lightwalletd-full-sync reads Zebra and writes lwd, so it is handled specially.
# TODO: we should find a better logic for this use cases
if: ${{ (inputs.needs_zebra_state && !inputs.needs_lwd_state) && inputs.test_id != 'lwd-full-sync' }}
id: attach-zebra-state
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker run \
--name ${{ inputs.test_id }} \
--tty \
--detach \
${{ inputs.test_variables }} \
--mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \
${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker run \
--name ${{ inputs.test_id }} \
--tty \
--detach \
${{ inputs.test_variables }} \
--mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \
${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \
"
# Launch the test with the previously created Lightwalletd and Zebra cached state.
# Each test runs one of the "Launch test" steps, and skips the other.
#
# SSH into the just created VM, and create a Docker container to run the incoming test
# from ${{ inputs.test_id }}, then mount the docker volume created in the previous job.
# from ${{ inputs.test_id }}, then mount the sudo docker volume created in the previous job.
#
# In this step we're using the same disk for simplicity, as mounting multiple disks to the
# VM and to the container might require more steps in this workflow, and additional
@ -491,21 +507,26 @@ jobs:
# lightwalletd-full-sync reads Zebra and writes lwd, so it is handled specially.
# TODO: we should find a better logic for this use cases
if: ${{ (inputs.needs_zebra_state && inputs.needs_lwd_state) || inputs.test_id == 'lwd-full-sync' }}
id: attach-multi-state
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker run \
--name ${{ inputs.test_id }} \
--tty \
--detach \
${{ inputs.test_variables }} \
--mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \
--mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.lwd_state_dir }} \
${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker run \
--name ${{ inputs.test_id }} \
--tty \
--detach \
${{ inputs.test_variables }} \
--mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \
--mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.lwd_state_dir }} \
${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \
"
# follow the logs of the test we just launched, up to Sapling activation (or the test finishing)
logs-sprout:
@ -547,31 +568,35 @@ jobs:
# Show all the logs since the container launched,
# following until Sapling activation (or the test finishes).
#
# The log pipeline ignores the exit status of `sudo docker logs`.
# The log pipeline ignores the exit status of `docker logs`.
# It also ignores the expected 'broken pipe' error from `tee`,
# which happens when `grep` finds a matching output and moves on to the next job.
#
# Errors in the tests are caught by the final test status job.
- name: Show logs for ${{ inputs.test_id }} test (sprout)
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*network_upgrade.*=.*Sapling' \
-e 'estimated progress.*network_upgrade.*=.*Blossom' \
-e 'estimated progress.*network_upgrade.*=.*Heartwood' \
-e 'estimated progress.*network_upgrade.*=.*Canopy' \
-e 'estimated progress.*network_upgrade.*=.*Nu5' \
-e 'test result:.*finished in'
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*network_upgrade.*=.*Sapling' \
-e 'estimated progress.*network_upgrade.*=.*Blossom' \
-e 'estimated progress.*network_upgrade.*=.*Heartwood' \
-e 'estimated progress.*network_upgrade.*=.*Canopy' \
-e 'estimated progress.*network_upgrade.*=.*Nu5' \
-e 'test result:.*finished in' \
"
# follow the logs of the test we just launched, up to Canopy activation (or the test finishing)
logs-heartwood:
@ -611,22 +636,26 @@ jobs:
# Show recent logs, following until Canopy activation (or the test finishes)
- name: Show logs for ${{ inputs.test_id }} test (heartwood)
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*network_upgrade.*=.*Canopy' \
-e 'estimated progress.*network_upgrade.*=.*Nu5' \
-e 'test result:.*finished in'
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*network_upgrade.*=.*Canopy' \
-e 'estimated progress.*network_upgrade.*=.*Nu5' \
-e 'test result:.*finished in' \
"
# follow the logs of the test we just launched, up to NU5 activation (or the test finishing)
logs-canopy:
@ -666,21 +695,25 @@ jobs:
# Show recent logs, following until NU5 activation (or the test finishes)
- name: Show logs for ${{ inputs.test_id }} test (canopy)
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*network_upgrade.*=.*Nu5' \
-e 'test result:.*finished in'
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*network_upgrade.*=.*Nu5' \
-e 'test result:.*finished in' \
"
# follow the logs of the test we just launched, up to block 1,740,000 or later
# (or the test finishing)
@ -723,23 +756,27 @@ jobs:
# Show recent logs, following until block 1,740,000 (or the test finishes)
- name: Show logs for ${{ inputs.test_id }} test (1740k)
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*current_height.*=.*17[4-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'test result:.*finished in'
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*current_height.*=.*17[4-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'test result:.*finished in' \
"
# follow the logs of the test we just launched, up to block 1,760,000 or later
# (or the test finishing)
@ -782,23 +819,27 @@ jobs:
# Show recent logs, following until block 1,760,000 (or the test finishes)
- name: Show logs for ${{ inputs.test_id }} test (1760k)
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*current_height.*=.*17[6-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'test result:.*finished in'
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*current_height.*=.*17[6-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'test result:.*finished in' \
"
# follow the logs of the test we just launched, up to block 1,780,000 or later
# (or the test finishing)
@ -841,23 +882,27 @@ jobs:
# Show recent logs, following until block 1,780,000 (or the test finishes)
- name: Show logs for ${{ inputs.test_id }} test (1780k)
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*current_height.*=.*17[8-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'test result:.*finished in'
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*current_height.*=.*17[8-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'test result:.*finished in' \
"
# follow the logs of the test we just launched, up to block 1,790,000 or later
# (or the test finishing)
@ -901,23 +946,27 @@ jobs:
# Show recent logs, following until block 1,790,000 (or the test finishes)
- name: Show logs for ${{ inputs.test_id }} test (1790k)
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*current_height.*=.*179[0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'test result:.*finished in'
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker logs \
--tail all \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'estimated progress.*current_height.*=.*179[0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \
-e 'test result:.*finished in' \
"
# follow the logs of the test we just launched, up to the last checkpoint (or the test finishing)
logs-checkpoint:
@ -959,21 +1008,25 @@ jobs:
#
# TODO: when doing obtain/extend tips, log the verifier in use, and check for full verification here
- name: Show logs for ${{ inputs.test_id }} test (checkpoint)
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs \
--tail ${{ env.EXTRA_LOG_LINES }} \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'verified final checkpoint' \
-e 'test result:.*finished in'
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker logs \
--tail ${{ env.EXTRA_LOG_LINES }} \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
-e 'verified final checkpoint' \
-e 'test result:.*finished in' \
"
# follow the logs of the test we just launched, until it finishes
logs-end:
@ -1013,20 +1066,24 @@ jobs:
# Show recent logs, following until the test finishes
- name: Show logs for ${{ inputs.test_id }} test (end)
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs \
--tail ${{ env.EXTRA_LOG_LINES }} \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
'test result:.*finished in'
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command \
"\
sudo docker logs \
--tail ${{ env.EXTRA_LOG_LINES }} \
--follow \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
'test result:.*finished in' \
"
# check the results of the test, and show all of the test logs
@ -1078,27 +1135,31 @@ jobs:
# with that status.
# (`docker wait` can also wait for multiple containers, but we only ever wait for a single container.)
- name: Result of ${{ inputs.test_id }} test
id: compute-ssh
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
set -e;
sudo docker logs \
--tail all \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
"test result: .*ok.* [1-9][0-9]* passed.*finished in"; \
EXIT_STATUS=$( \
sudo docker wait ${{ inputs.test_id }} || \
sudo docker inspect --format "{{.State.ExitCode}}" ${{ inputs.test_id }} || \
echo "missing container, or missing exit status for container" \
); \
echo "sudo docker exit status: $EXIT_STATUS"; \
exit "$EXIT_STATUS"
run: |
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command=' \
set -e;
sudo docker logs \
--tail all \
${{ inputs.test_id }} | \
tee --output-error=exit /dev/stderr | \
grep --max-count=1 --extended-regexp --color=always \
"test result: .*ok.* [1-9][0-9]* passed.*finished in"; \
EXIT_STATUS=$( \
sudo docker wait ${{ inputs.test_id }} || \
sudo docker inspect --format "{{.State.ExitCode}}" ${{ inputs.test_id }} || \
echo "missing container, or missing exit status for container" \
); \
echo "sudo docker exit status: $EXIT_STATUS"; \
exit "$EXIT_STATUS" \
'
# create a state image from the instance's state disk, if requested by the caller
create-state-image:
@ -1185,16 +1246,7 @@ jobs:
# Get the sync height from the test logs, which is later used as part of the
# disk description and labels.
- name: Get sync height from logs
id: get-sync-height
uses: google-github-actions/ssh-compute@v0.1.2
with:
instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}
zone: ${{ env.ZONE }}
ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }}
command: |
sudo docker logs ${{ inputs.test_id }} --tail 200
#
# The regex used to grep the sync height is provided by ${{ inputs.height_grep_text }},
# this allows to dynamically change the height as needed by different situations or
# based on the logs output from different tests.
@ -1206,8 +1258,20 @@ jobs:
run: |
SYNC_HEIGHT=""
DOCKER_LOGS=$( \
gcloud compute ssh \
github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--ssh-flag="-o ConnectionAttempts=20" \
--ssh-flag="-o ConnectTimeout=5" \
--command=" \
docker logs ${{ inputs.test_id }} --tail 200 \
")
SYNC_HEIGHT=$( \
echo "${{ steps.get-sync-height.outputs.stdout }}" | \
echo "$DOCKER_LOGS" | \
grep --extended-regexp --only-matching '${{ inputs.height_grep_text }}[0-9]+' | \
grep --extended-regexp --only-matching '[0-9]+' | \
tail -1 || \

4
.github/workflows/lint.yml vendored
View File

@ -35,13 +35,12 @@ jobs:
- uses: actions/checkout@v3.1.0
with:
persist-credentials: false
fetch-depth: 2
fetch-depth: 0
- name: Rust files
id: changed-files-rust
uses: tj-actions/changed-files@v32.0.0
with:
fetch-depth: 2
files: |
**/*.rs
**/Cargo.toml
@ -54,7 +53,6 @@ jobs:
id: changed-files-workflows
uses: tj-actions/changed-files@v32.0.0
with:
fetch-depth: 2
files: |
.github/workflows/*.yml

22
.github/workflows/scripts/gcp-vm-startup-script.sh vendored Executable file
View File

@ -0,0 +1,22 @@
#! /bin/bash
# Increase the Google Cloud instance sshd connection limit
#
# This script appends 'MaxStartups 500' to /etc/ssh/sshd_config allowing up to 500
# unauthenticated connections to Google Cloud instances.
ps auxwww | grep sshd
echo
sudo grep MaxStartups /etc/ssh/sshd_config
echo 'Original config:'
sudo cat /etc/ssh/sshd_config
echo
echo 'Modifying config:'
echo 'MaxStartups 500' | sudo tee --append /etc/ssh/sshd_config \
|| \
(echo "updating instance sshd config failed: failing test"; exit 1)
sudo grep MaxStartups /etc/ssh/sshd_config
echo 'Modified config:'
sudo cat /etc/ssh/sshd_config
echo
sudo systemctl reload sshd.service
echo
ps auxwww | grep sshd