diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index e0ea6e3ef..f0ecbdbbc 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -1175,8 +1175,7 @@ jobs: # follow the logs of the test we just launched, up to block 1,880,000 or later # (or the test finishing) # - # We chose this height because it should be about 5 hours from the last job, - # but if that's not the case we'll need to fix it. + # We chose this height because it was about 5 hours from the last job, in December 2022. logs-1880k: name: Log ${{ inputs.test_id }} test (1880k) needs: [ logs-1850k ] @@ -1242,10 +1241,146 @@ jobs: -e 'test result:.*finished in' \ " + # follow the logs of the test we just launched, up to block 1,920,000 or later + # (or the test finishing) + # + # We chose this height because it was about 4 hours from the last job, in February 2023. + logs-1920k: + name: Log ${{ inputs.test_id }} test (1920k) + needs: [ logs-1880k ] + # If the previous job fails, we still want to show the logs. + if: ${{ !cancelled() && inputs.is_long_test }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.3.0 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + # Install our SSH secret + - name: Install private SSH key + uses: shimataro/ssh-key-action@v2.5.0 + with: + key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} + name: google_compute_engine + known_hosts: unnecessary + + - name: Generate public SSH key + run: ssh-keygen -y -f ~/.ssh/google_compute_engine > ~/.ssh/google_compute_engine.pub + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v1.0.0 + with: + retries: '3' + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1.0.1 + + # Show recent logs, following until block 1,920,000 (or the test finishes) + - name: Show logs for ${{ inputs.test_id }} test (1920k) + run: | + gcloud compute ssh ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*current_height.*=.*19[2-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'test result:.*finished in' \ + " + + # follow the logs of the test we just launched, up to block 1,960,000 or later + # (or the test finishing) + # + # We chose this height because it was about 4 hours from the last job, in February 2023. + logs-1960k: + name: Log ${{ inputs.test_id }} test (1960k) + needs: [ logs-1920k ] + # If the previous job fails, we still want to show the logs. + if: ${{ !cancelled() && inputs.is_long_test }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.3.0 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + # Install our SSH secret + - name: Install private SSH key + uses: shimataro/ssh-key-action@v2.5.0 + with: + key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} + name: google_compute_engine + known_hosts: unnecessary + + - name: Generate public SSH key + run: ssh-keygen -y -f ~/.ssh/google_compute_engine > ~/.ssh/google_compute_engine.pub + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v1.0.0 + with: + retries: '3' + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1.0.1 + + # Show recent logs, following until block 1,960,000 (or the test finishes) + - name: Show logs for ${{ inputs.test_id }} test (1920k) + run: | + gcloud compute ssh ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*current_height.*=.*19[6-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'test result:.*finished in' \ + " + # follow the logs of the test we just launched, up to the last checkpoint (or the test finishing) logs-checkpoint: name: Log ${{ inputs.test_id }} test (checkpoint) - needs: [ logs-1880k ] + needs: [ logs-1960k ] # If the previous job fails, we still want to show the logs. if: ${{ !cancelled() && inputs.is_long_test }} runs-on: ubuntu-latest diff --git a/zebrad/tests/common/sync.rs b/zebrad/tests/common/sync.rs index 806b546e9..72103b081 100644 --- a/zebrad/tests/common/sync.rs +++ b/zebrad/tests/common/sync.rs @@ -74,7 +74,7 @@ pub const FINISH_PARTIAL_SYNC_TIMEOUT: Duration = Duration::from_secs(11 * 60 * /// The maximum time to wait for Zebrad to synchronize up to the chain tip starting from the /// genesis block. -pub const FINISH_FULL_SYNC_TIMEOUT: Duration = Duration::from_secs(48 * 60 * 60); +pub const FINISH_FULL_SYNC_TIMEOUT: Duration = Duration::from_secs(58 * 60 * 60); /// The test sync height where we switch to using the default lookahead limit. ///