fix(ci): Retry after docker log follow ssh failures (#4198)

* Retry after log ssh failures in full sync test

* Retry after log failures in other docker tests
This commit is contained in:
teor 2022-04-27 07:01:24 +10:00 committed by GitHub
parent d476c18339
commit 4f92df42b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 18 deletions

View File

@ -236,12 +236,15 @@ jobs:
- name: Full sync
id: full-sync
run: |
gcloud compute ssh \
full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--command="docker logs --follow ${{ env.CONTAINER_NAME }}"
for RETRY in 1 2 3 4; do
gcloud compute ssh \
full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=15" \
--command="docker logs --follow ${{ env.CONTAINER_NAME }}" \
|| echo "ssh disconnected $RETRY times"
done
EXIT_CODE=$(\
gcloud compute ssh \

View File

@ -347,12 +347,15 @@ jobs:
id: sync-to-checkpoint
if: ${{ steps.create-instance.outcome == 'success' }}
run: |
gcloud compute ssh \
regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--command="docker logs --follow ${{ env.CONTAINER_NAME }}"
for RETRY in 1 2 3 4; do
gcloud compute ssh \
regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=15" \
--command="docker logs --follow ${{ env.CONTAINER_NAME }}" \
|| echo "ssh disconnected $RETRY times"
done
EXIT_CODE=$(\
gcloud compute ssh \
@ -512,12 +515,15 @@ jobs:
- name: Sync past mandatory checkpoint
id: sync-past-checkpoint
run: |
gcloud compute ssh \
sync-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=5" \
--command="docker logs --follow ${{ env.CONTAINER_NAME }}"
for RETRY in 1 2 3 4; do
gcloud compute ssh \
sync-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
--zone ${{ env.ZONE }} \
--quiet \
--ssh-flag="-o ServerAliveInterval=15" \
--command="docker logs --follow ${{ env.CONTAINER_NAME }}" \
|| echo "ssh disconnected $RETRY times"
done
EXIT_CODE=$(\
gcloud compute ssh \