From 658fbd923a848b122aac2df089522bc962d759a5 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 10 Oct 2022 20:11:49 -0400 Subject: [PATCH] ci(ssh): revert using `ssh-compute` action & increase sshd connection limit (#5367) * Revert "ci(ssh): connect using `ssh-compute` action by Google (#5330)" This reverts commit b366d6e7bb41125fbb861aa551610b3c3de7a544. * ci(ssh): use sudo for docker commands if user is not root * ci(ssh): specify the service account to connect with * ci(ssh): increase the Google Cloud instance sshd connection limit * chore: add a new line at the end of the script * chore: update our VM image to bullseye * chore: fix `tj-actions/changed-files` file comparison --- .../continous-integration-docker.yml | 9 - .github/workflows/deploy-gcp-tests.yml | 572 ++++++++++-------- .github/workflows/lint.yml | 4 +- .../scripts/gcp-vm-startup-script.sh | 22 + 4 files changed, 341 insertions(+), 266 deletions(-) create mode 100755 .github/workflows/scripts/gcp-vm-startup-script.sh diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 065bc3a55..9f6c327f6 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -281,7 +281,6 @@ jobs: saves_to_disk: true disk_suffix: checkpoint height_grep_text: 'flushing database to disk .*height.*=.*Height.*\(' - secrets: inherit # Test that Zebra syncs and fully validates a few thousand blocks from a cached mandatory checkpoint disk # @@ -300,7 +299,6 @@ jobs: needs_zebra_state: true saves_to_disk: false disk_suffix: checkpoint - secrets: inherit # zebrad cached tip state tests @@ -330,7 +328,6 @@ jobs: saves_to_disk: true disk_suffix: tip height_grep_text: 'current_height.*=.*Height.*\(' - secrets: inherit # We don't want to cancel running full syncs on `main` if a new PR gets merged, # because we might never finish a full sync during busy weeks. Instead, we let the # first sync complete, then queue the latest pending sync, cancelling any syncs in between. @@ -370,7 +367,6 @@ jobs: root_state_path: '/var/cache' zebra_state_dir: 'zebrad-cache' height_grep_text: 'current_height.*=.*Height.*\(' - secrets: inherit # lightwalletd cached tip state tests @@ -404,7 +400,6 @@ jobs: zebra_state_dir: 'zebrad-cache' lwd_state_dir: 'lwd-cache' height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )' - secrets: inherit # Test update sync of lightwalletd with a lightwalletd and Zebra tip state # Runs: @@ -433,7 +428,6 @@ jobs: zebra_state_dir: 'zebrad-cache' lwd_state_dir: 'lwd-cache' height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )' - secrets: inherit # Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state # @@ -458,7 +452,6 @@ jobs: disk_suffix: tip root_state_path: '/var/cache' zebra_state_dir: 'zebrad-cache' - secrets: inherit # Test that Zebra can handle a lightwalletd send transaction RPC call, using a cached Zebra tip state # @@ -486,7 +479,6 @@ jobs: root_state_path: '/var/cache' zebra_state_dir: 'zebrad-cache' lwd_state_dir: 'lwd-cache' - secrets: inherit # Test that Zebra can handle gRPC wallet calls, using a cached Zebra tip state # @@ -513,4 +505,3 @@ jobs: root_state_path: '/var/cache' zebra_state_dir: 'zebrad-cache' lwd_state_dir: 'lwd-cache' - secrets: inherit diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index f752a6997..9d43a853c 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -79,9 +79,6 @@ on: type: string default: 'zebra' description: 'Application name, used to work out when a job is an update job' - secrets: - GCP_SSH_PRIVATE_KEY: - required: true env: # where we get the Docker image from @@ -145,11 +142,12 @@ jobs: --boot-disk-size 200GB \ --boot-disk-type pd-ssd \ --create-disk name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=200GB,type=pd-ssd \ - --container-image debian:buster \ + --container-image debian:bullseye \ --container-restart-policy=never \ --machine-type ${{ env.MACHINE_TYPE }} \ --scopes cloud-platform \ --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ + --metadata-from-file=startup-script=.github/workflows/scripts/gcp-vm-startup-script.sh \ --tags ${{ inputs.app_name }} \ --zone ${{ env.ZONE }} sleep 60 @@ -158,17 +156,21 @@ jobs: # # SSH into the just created VM, and create a docker volume with the newly created disk. - name: Create ${{ inputs.test_id }} Docker volume - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo mkfs.ext4 -v /dev/sdb \ - && \ - sudo docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ - ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo mkfs.ext4 -v /dev/sdb \ + && \ + sudo docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ + ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ + " # launch the test, if it doesn't use any cached state launch-without-cached-state: @@ -208,20 +210,25 @@ jobs: # Launch the test without any cached state - name: Launch ${{ inputs.test_id }} test - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker run \ - --name ${{ inputs.test_id }} \ - --tty \ - --detach \ - ${{ inputs.test_variables }} \ - --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ - ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker run \ + --name ${{ inputs.test_id }} \ + --tty \ + --detach \ + ${{ inputs.test_variables }} \ + --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \ + " + # set up the test, if it uses cached state # each test runs one of the *-with/without-cached-state job series, and skips the other @@ -357,11 +364,12 @@ jobs: --boot-disk-size 200GB \ --boot-disk-type pd-ssd \ --create-disk image=${{ env.CACHED_DISK_NAME }},name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=200GB,type=pd-ssd \ - --container-image debian:buster \ + --container-image debian:bullseye \ --container-restart-policy=never \ --machine-type ${{ env.MACHINE_TYPE }} \ --scopes cloud-platform \ --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ + --metadata-from-file=startup-script=.github/workflows/scripts/gcp-vm-startup-script.sh \ --tags ${{ inputs.app_name }} \ --zone ${{ env.ZONE }} sleep 60 @@ -373,19 +381,23 @@ jobs: # (The cached state and disk are usually the same size, # but the cached state can be smaller if we just increased the disk size.) - name: Create ${{ inputs.test_id }} Docker volume - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo e2fsck -v -f -p /dev/sdb \ - && \ - sudo resize2fs -p /dev/sdb \ - && \ - sudo docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ - ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo e2fsck -v -f -p /dev/sdb \ + && \ + sudo resize2fs -p /dev/sdb \ + && \ + sudo docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ + ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ + " # launch the test, if it uses cached state launch-with-cached-state: @@ -427,7 +439,7 @@ jobs: # Each test runs one of the "Launch test" steps, and skips the other. # # SSH into the just created VM, and create a Docker container to run the incoming test - # from ${{ inputs.test_id }}, then mount the docker volume created in the previous job. + # from ${{ inputs.test_id }}, then mount the sudo docker volume created in the previous job. # # The disk mounted in the VM is located at /dev/sdb, we mount the root `/` of this disk to the docker # container in one path: @@ -445,26 +457,30 @@ jobs: # lightwalletd-full-sync reads Zebra and writes lwd, so it is handled specially. # TODO: we should find a better logic for this use cases if: ${{ (inputs.needs_zebra_state && !inputs.needs_lwd_state) && inputs.test_id != 'lwd-full-sync' }} - id: attach-zebra-state - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker run \ - --name ${{ inputs.test_id }} \ - --tty \ - --detach \ - ${{ inputs.test_variables }} \ - --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ - ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker run \ + --name ${{ inputs.test_id }} \ + --tty \ + --detach \ + ${{ inputs.test_variables }} \ + --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \ + " # Launch the test with the previously created Lightwalletd and Zebra cached state. # Each test runs one of the "Launch test" steps, and skips the other. # # SSH into the just created VM, and create a Docker container to run the incoming test - # from ${{ inputs.test_id }}, then mount the docker volume created in the previous job. + # from ${{ inputs.test_id }}, then mount the sudo docker volume created in the previous job. # # In this step we're using the same disk for simplicity, as mounting multiple disks to the # VM and to the container might require more steps in this workflow, and additional @@ -491,21 +507,26 @@ jobs: # lightwalletd-full-sync reads Zebra and writes lwd, so it is handled specially. # TODO: we should find a better logic for this use cases if: ${{ (inputs.needs_zebra_state && inputs.needs_lwd_state) || inputs.test_id == 'lwd-full-sync' }} - id: attach-multi-state - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker run \ - --name ${{ inputs.test_id }} \ - --tty \ - --detach \ - ${{ inputs.test_variables }} \ - --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ - --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.lwd_state_dir }} \ - ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker run \ + --name ${{ inputs.test_id }} \ + --tty \ + --detach \ + ${{ inputs.test_variables }} \ + --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ + --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.lwd_state_dir }} \ + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \ + " + # follow the logs of the test we just launched, up to Sapling activation (or the test finishing) logs-sprout: @@ -547,31 +568,35 @@ jobs: # Show all the logs since the container launched, # following until Sapling activation (or the test finishes). # - # The log pipeline ignores the exit status of `sudo docker logs`. + # The log pipeline ignores the exit status of `docker logs`. # It also ignores the expected 'broken pipe' error from `tee`, # which happens when `grep` finds a matching output and moves on to the next job. # # Errors in the tests are caught by the final test status job. - name: Show logs for ${{ inputs.test_id }} test (sprout) - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs \ - --tail all \ - --follow \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e 'estimated progress.*network_upgrade.*=.*Sapling' \ - -e 'estimated progress.*network_upgrade.*=.*Blossom' \ - -e 'estimated progress.*network_upgrade.*=.*Heartwood' \ - -e 'estimated progress.*network_upgrade.*=.*Canopy' \ - -e 'estimated progress.*network_upgrade.*=.*Nu5' \ - -e 'test result:.*finished in' + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*network_upgrade.*=.*Sapling' \ + -e 'estimated progress.*network_upgrade.*=.*Blossom' \ + -e 'estimated progress.*network_upgrade.*=.*Heartwood' \ + -e 'estimated progress.*network_upgrade.*=.*Canopy' \ + -e 'estimated progress.*network_upgrade.*=.*Nu5' \ + -e 'test result:.*finished in' \ + " # follow the logs of the test we just launched, up to Canopy activation (or the test finishing) logs-heartwood: @@ -611,22 +636,26 @@ jobs: # Show recent logs, following until Canopy activation (or the test finishes) - name: Show logs for ${{ inputs.test_id }} test (heartwood) - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs \ - --tail all \ - --follow \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e 'estimated progress.*network_upgrade.*=.*Canopy' \ - -e 'estimated progress.*network_upgrade.*=.*Nu5' \ - -e 'test result:.*finished in' + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*network_upgrade.*=.*Canopy' \ + -e 'estimated progress.*network_upgrade.*=.*Nu5' \ + -e 'test result:.*finished in' \ + " # follow the logs of the test we just launched, up to NU5 activation (or the test finishing) logs-canopy: @@ -666,21 +695,25 @@ jobs: # Show recent logs, following until NU5 activation (or the test finishes) - name: Show logs for ${{ inputs.test_id }} test (canopy) - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs \ - --tail all \ - --follow \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e 'estimated progress.*network_upgrade.*=.*Nu5' \ - -e 'test result:.*finished in' + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*network_upgrade.*=.*Nu5' \ + -e 'test result:.*finished in' \ + " # follow the logs of the test we just launched, up to block 1,740,000 or later # (or the test finishing) @@ -723,23 +756,27 @@ jobs: # Show recent logs, following until block 1,740,000 (or the test finishes) - name: Show logs for ${{ inputs.test_id }} test (1740k) - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs \ - --tail all \ - --follow \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e 'estimated progress.*current_height.*=.*17[4-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'test result:.*finished in' + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*current_height.*=.*17[4-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'test result:.*finished in' \ + " # follow the logs of the test we just launched, up to block 1,760,000 or later # (or the test finishing) @@ -782,23 +819,27 @@ jobs: # Show recent logs, following until block 1,760,000 (or the test finishes) - name: Show logs for ${{ inputs.test_id }} test (1760k) - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs \ - --tail all \ - --follow \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e 'estimated progress.*current_height.*=.*17[6-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'test result:.*finished in' + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*current_height.*=.*17[6-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'test result:.*finished in' \ + " # follow the logs of the test we just launched, up to block 1,780,000 or later # (or the test finishing) @@ -841,23 +882,27 @@ jobs: # Show recent logs, following until block 1,780,000 (or the test finishes) - name: Show logs for ${{ inputs.test_id }} test (1780k) - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs \ - --tail all \ - --follow \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e 'estimated progress.*current_height.*=.*17[8-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'test result:.*finished in' + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*current_height.*=.*17[8-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'test result:.*finished in' \ + " # follow the logs of the test we just launched, up to block 1,790,000 or later # (or the test finishing) @@ -901,23 +946,27 @@ jobs: # Show recent logs, following until block 1,790,000 (or the test finishes) - name: Show logs for ${{ inputs.test_id }} test (1790k) - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs \ - --tail all \ - --follow \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e 'estimated progress.*current_height.*=.*179[0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ - -e 'test result:.*finished in' + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*current_height.*=.*179[0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'test result:.*finished in' \ + " # follow the logs of the test we just launched, up to the last checkpoint (or the test finishing) logs-checkpoint: @@ -959,21 +1008,25 @@ jobs: # # TODO: when doing obtain/extend tips, log the verifier in use, and check for full verification here - name: Show logs for ${{ inputs.test_id }} test (checkpoint) - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs \ - --tail ${{ env.EXTRA_LOG_LINES }} \ - --follow \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e 'verified final checkpoint' \ - -e 'test result:.*finished in' + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail ${{ env.EXTRA_LOG_LINES }} \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'verified final checkpoint' \ + -e 'test result:.*finished in' \ + " # follow the logs of the test we just launched, until it finishes logs-end: @@ -1013,20 +1066,24 @@ jobs: # Show recent logs, following until the test finishes - name: Show logs for ${{ inputs.test_id }} test (end) - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs \ - --tail ${{ env.EXTRA_LOG_LINES }} \ - --follow \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - 'test result:.*finished in' + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail ${{ env.EXTRA_LOG_LINES }} \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + 'test result:.*finished in' \ + " # check the results of the test, and show all of the test logs @@ -1078,27 +1135,31 @@ jobs: # with that status. # (`docker wait` can also wait for multiple containers, but we only ever wait for a single container.) - name: Result of ${{ inputs.test_id }} test - id: compute-ssh - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - set -e; - sudo docker logs \ - --tail all \ - ${{ inputs.test_id }} | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - "test result: .*ok.* [1-9][0-9]* passed.*finished in"; \ - EXIT_STATUS=$( \ - sudo docker wait ${{ inputs.test_id }} || \ - sudo docker inspect --format "{{.State.ExitCode}}" ${{ inputs.test_id }} || \ - echo "missing container, or missing exit status for container" \ - ); \ - echo "sudo docker exit status: $EXIT_STATUS"; \ - exit "$EXIT_STATUS" + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command=' \ + set -e; + sudo docker logs \ + --tail all \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + "test result: .*ok.* [1-9][0-9]* passed.*finished in"; \ + EXIT_STATUS=$( \ + sudo docker wait ${{ inputs.test_id }} || \ + sudo docker inspect --format "{{.State.ExitCode}}" ${{ inputs.test_id }} || \ + echo "missing container, or missing exit status for container" \ + ); \ + echo "sudo docker exit status: $EXIT_STATUS"; \ + exit "$EXIT_STATUS" \ + ' + # create a state image from the instance's state disk, if requested by the caller create-state-image: @@ -1185,16 +1246,7 @@ jobs: # Get the sync height from the test logs, which is later used as part of the # disk description and labels. - - name: Get sync height from logs - id: get-sync-height - uses: google-github-actions/ssh-compute@v0.1.2 - with: - instance_name: ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} - zone: ${{ env.ZONE }} - ssh_private_key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - command: | - sudo docker logs ${{ inputs.test_id }} --tail 200 - + # # The regex used to grep the sync height is provided by ${{ inputs.height_grep_text }}, # this allows to dynamically change the height as needed by different situations or # based on the logs output from different tests. @@ -1206,8 +1258,20 @@ jobs: run: | SYNC_HEIGHT="" + DOCKER_LOGS=$( \ + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command=" \ + docker logs ${{ inputs.test_id }} --tail 200 \ + ") + SYNC_HEIGHT=$( \ - echo "${{ steps.get-sync-height.outputs.stdout }}" | \ + echo "$DOCKER_LOGS" | \ grep --extended-regexp --only-matching '${{ inputs.height_grep_text }}[0-9]+' | \ grep --extended-regexp --only-matching '[0-9]+' | \ tail -1 || \ diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0bea8837b..fe1b426df 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,13 +35,12 @@ jobs: - uses: actions/checkout@v3.1.0 with: persist-credentials: false - fetch-depth: 2 + fetch-depth: 0 - name: Rust files id: changed-files-rust uses: tj-actions/changed-files@v32.0.0 with: - fetch-depth: 2 files: | **/*.rs **/Cargo.toml @@ -54,7 +53,6 @@ jobs: id: changed-files-workflows uses: tj-actions/changed-files@v32.0.0 with: - fetch-depth: 2 files: | .github/workflows/*.yml diff --git a/.github/workflows/scripts/gcp-vm-startup-script.sh b/.github/workflows/scripts/gcp-vm-startup-script.sh new file mode 100755 index 000000000..da65ff267 --- /dev/null +++ b/.github/workflows/scripts/gcp-vm-startup-script.sh @@ -0,0 +1,22 @@ +#! /bin/bash +# Increase the Google Cloud instance sshd connection limit +# +# This script appends 'MaxStartups 500' to /etc/ssh/sshd_config allowing up to 500 +# unauthenticated connections to Google Cloud instances. +ps auxwww | grep sshd +echo +sudo grep MaxStartups /etc/ssh/sshd_config +echo 'Original config:' +sudo cat /etc/ssh/sshd_config +echo +echo 'Modifying config:' +echo 'MaxStartups 500' | sudo tee --append /etc/ssh/sshd_config \ +|| \ +(echo "updating instance sshd config failed: failing test"; exit 1) +sudo grep MaxStartups /etc/ssh/sshd_config +echo 'Modified config:' +sudo cat /etc/ssh/sshd_config +echo +sudo systemctl reload sshd.service +echo +ps auxwww | grep sshd