Run acceptance tests post-merge with large cached state (#1282)

* Create and mount persistent disk to store zebrad state, update runner container config to use * Enable checkpoint sync in zebrad image config * Lower state memory cache from 500MB to 50MB * Upgrade host to n2-standard-4 * Bump zebrad-cache disk size to 100GB * Copy zebrad as the tests are compiled with a hardcoded path to it * Rename all debug binaries for easy invocation * Name state cache disk, use the correct path to binaries * Create volume and all that jazz on instance creation Otherwise there's a lot of on-instance commands to do that is just handled by this shortcut. * Explicitly mount the state cache and cleanup test instance * Wait for zebra-test container to start then attach * Always clean up even if the tests step fails * Keep fast sleep but only print 'waiting' once
2020-11-12 15:18:35 -05:00 · 2020-11-12 15:18:35 -05:00 · 8b5b1b49cc
parent 32934bd574
commit 8b5b1b49cc
3 changed files with 165 additions and 50 deletions
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@ -10,26 +10,25 @@ env:
  PROJECT_ID: zealous-zebra

 jobs:
+
  build:
-    name: Google Cloud / Test, Build, and Deploy
+    name: Build images
    timeout-minutes: 30
    runs-on: ubuntu-latest
    steps:
-
-    - name: Checkout
-      uses: actions/checkout@master
+    - uses: actions/checkout@v2

    - name: Set project and image names
      run: |
        BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
        BRANCH_NAME=${BRANCH_NAME,,} && \
        REPOSITORY=${GITHUB_REPOSITORY,,} && \
-        echo "::set-env name=BRANCH_NAME::$BRANCH_NAME" && \
-        echo "::set-env name=SHORT_SHA::$(git rev-parse --short=7 $GITHUB_SHA)" && \
-        echo "::set-env name=REPOSITORY::$REPOSITORY"
+        echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
+        echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
+        echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV

    # Setup gcloud CLI
-    - name: Set up gcloud Cloud SDK environment
+    - name: Set up gcloud SDK environment
      uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
      with:
        version: '295.0.0'
@ -44,46 +43,135 @@ jobs:
          --config cloudbuild.yaml \
          --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME"

-    # Create instance template from container image
-    - name: Create instance template
-      run: |
-        gcloud compute instance-templates create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \
+  test:
+    name: Run integration tests
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set project and image names
+        run: |
+          BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
+          BRANCH_NAME=${BRANCH_NAME,,} && \
+          REPOSITORY=${GITHUB_REPOSITORY,,} && \
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
+          echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
+          echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
+
+      # Setup gcloud CLI
+      - name: Set up gcloud SDK environment
+        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          version: '295.0.0'
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+      # Run once: create firewall rule to allow incoming traffic to the nodes
+      # - name: Create Zcash incoming traffic firewall rule
+      #   run: |
+      #     gcloud compute firewall-rules create "allow-zcash" \
+      #     --target-tags zebrad \
+      #     --allow tcp:8233,tcp:18233 \
+      #     --source-ranges 0.0.0.0/0 \
+      #     --description="Allow incoming Zcash traffic from anywhere" \
+
+      # Creates Compute Engine virtual machine instance w/ test container and disks
+      - name: Create instance with test container image
+        run: |
+          gcloud compute instances create-with-container "zebra-tests-$BRANCH_NAME-$SHORT_SHA" \
+          --container-command='/zebra/target/debug/deps/acceptance' \
+          --container-arg='-Z' --container-arg='unstable-options' --container-arg='--include-ignored' \
+          --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/tests/$BRANCH_NAME:$SHORT_SHA" \
+          --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \
+          --container-restart-policy='never' \
+          --create-disk name=zebrad-cache-$SHORT_SHA,image=zebrad-cache-mainnet-height-1018849 \
+          --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
+          --scopes cloud-platform \
+          --tags zebrad \
+          --zone us-central1-a
+
+      # Wait for acceptance test container is up and stream its results
+      - name: Run acceptance tests
+        run: |
+          gcloud compute ssh "zebra-tests-$BRANCH_NAME-$SHORT_SHA" --zone us-central1-a \
+          --command "echo 'Pulling container image...'; until [[ -n \$(docker ps --all -q --filter name=zebra-test) ]]; do sleep 0.1; done; docker attach \$(docker ps -q -a --filter name=zebra-test)"
+
+      # Clean up
+      - name: Delete test instance
+        # Always run even if the earlier step fails
+        if: ${{ always() }}
+        run: |
+          gcloud compute instances delete "zebra-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone us-central1-a
+
+  deploy:
+    name: Deploy mainnet nodes
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set project and image names
+        run: |
+          BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
+          BRANCH_NAME=${BRANCH_NAME,,} && \
+          REPOSITORY=${GITHUB_REPOSITORY,,} && \
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
+          echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
+          echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
+
+      # Setup gcloud CLI
+      - name: Set up gcloud SDK environment
+        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          version: '295.0.0'
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+      # Run once: create firewall rule to allow incoming traffic to the nodes
+      # - name: Create Zcash incoming traffic firewall rule
+      #   run: |
+      #     gcloud compute firewall-rules create "allow-zcash" \
+      #     --target-tags zebrad \
+      #     --allow tcp:8233,tcp:18233 \
+      #     --source-ranges 0.0.0.0/0 \
+      #     --description="Allow incoming Zcash traffic from anywhere" \
+
+      # Create instance template from container image
+      - name: Create instance template
+        run: |
+          gcloud compute instance-templates create-with-container "zebrad-$BRANCH_NAME-$SHORT_SHA" \
          --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
-          --machine-type n1-highcpu-2 \
+          --machine-type n2-standard-4 \
+          --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced,auto-delete=no \
+          --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-$SHORT_SHA \
          --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
          --scopes cloud-platform \
          --tags zebrad \

-    # Run once: create firewall rule to allow healthchecks
-    # - name: Create healthcheck firewall rule
-    #   run: |
-    #     gcloud compute firewall-rules create "allow-tracing-health-check" \
-    #     --target-tags zebrad \
-    #     --allow tcp:3000 \
-    #     --source-ranges 130.211.0.0/22,35.191.0.0/16 \
-    #     --description="Allow HTTP requests to our tracing endpoint from Google's probes" \
+      # Run once: create firewall rule to allow healthchecks
+      # - name: Create healthcheck firewall rule
+      #   run: |
+      #     gcloud compute firewall-rules create "allow-tracing-health-check" \
+      #     --target-tags zebrad \
+      #     --allow tcp:3000 \
+      #     --source-ranges 130.211.0.0/22,35.191.0.0/16 \
+      #     --description="Allow HTTP requests to our tracing endpoint from Google's probes" \

-    # Run once: create firewall rule to allow incoming traffic to the nodes
-    # - name: Create Zcash incoming traffic firewall rule
-    #   run: |
-    #     gcloud compute firewall-rules create "allow-zcash" \
-    #     --target-tags zebrad \
-    #     --allow tcp:8233,tcp:18233 \
-    #     --source-ranges 0.0.0.0/0 \
-    #     --description="Allow incoming Zcash traffic from anywhere" \
+      # Check if our destination instance group exists already
+      - name: Check if instance group exists
+        id: does-group-exist
+        continue-on-error: true
+        run: |
+          gcloud compute instance-groups list | grep "zebrad-$BRANCH_NAME"

-    # Check if our destination instance group exists already
-    - name: Check if instance group exists
-      id: does-group-exist
-      continue-on-error: true
-      run: |
-        gcloud compute instance-groups list | grep "zebrad-$BRANCH_NAME"
-
-    # Deploy new managed instance group using the new instance template
-    - name: Create managed instance group
-      if: steps.does-group-exist.outcome == 'failure'
-      run: |
-        gcloud compute instance-groups managed create \
+      # Deploy new managed instance group using the new instance template
+      - name: Create managed instance group
+        if: steps.does-group-exist.outcome == 'failure'
+        run: |
+          gcloud compute instance-groups managed create \
          "zebrad-$BRANCH_NAME" \
          --template "zebrad-$BRANCH_NAME-$SHORT_SHA" \
          --health-check zebrad-tracing-filter \
@ -91,11 +179,11 @@ jobs:
          --region us-central1 \
          --size 2

-    # Rolls out update to existing group using the new instance template
-    - name: Update managed instance group
-      if: steps.does-group-exist.outcome == 'success'
-      run: |
-        gcloud compute instance-groups managed rolling-action start-update \
+      # Rolls out update to existing group using the new instance template
+      - name: Update managed instance group
+        if: steps.does-group-exist.outcome == 'success'
+        run: |
+          gcloud compute instance-groups managed rolling-action start-update \
          "zebrad-$BRANCH_NAME" \
          --version template="zebrad-$BRANCH_NAME-$SHORT_SHA" \
          --region us-central1 \
--- a/27
+++ b/27
@ -1,3 +1,4 @@
+# Builder image
 FROM rust:buster as builder

 RUN apt-get update && \
@ -16,10 +17,32 @@ COPY . .
 RUN cargo fetch --verbose
 COPY . .
 RUN rustc -V; cargo -V; rustup -V; cargo test --all && cargo build --release
+RUN find /zebra/target/debug/deps -type f -perm 755 ! -name '*.dylib' ! -name '*.so' | sed -e 'p;s/-.*//' | xargs -n2 mv


-FROM debian:buster-slim
+# Test binaries image
+FROM debian:buster AS zebra-tests
+
+RUN mkdir /zebra
+WORKDIR /zebra
+COPY --from=builder /zebra/target/debug/zebrad /zebra/target/debug/zebrad
+COPY --from=builder /zebra/target/debug/deps/[a-z0-9_]* /zebra/target/debug/deps/
+EXPOSE 8233 18233
+
+
+# Runner image
+FROM debian:buster-slim AS zebrad-release
+
 COPY --from=builder /zebra/target/release/zebrad /
-RUN echo "[tracing]\nendpoint_addr = '0.0.0.0:3000'" > /zebrad.toml
+
+RUN printf "[consensus]\n" >> /zebrad.toml
+RUN printf "checkpoint_sync = true\n" >> /zebrad.toml
+RUN printf "[state]\n" >> /zebrad.toml
+RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
+RUN printf "memory_cache_bytes = 52428800\n" >> /zebrad.toml
+RUN printf "[tracing]\n" >> /zebrad.toml
+RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml
+
 EXPOSE 3000 8233 18233
+
 CMD [ "/zebrad", "-c", "/zebrad.toml", "start" ]
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@ -1,8 +1,12 @@
 steps:
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['build', '-t', 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA', '.']
+  args: ['build', '--target', 'zebra-tests', '-t', 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/tests/$BRANCH_NAME:$SHORT_SHA', '.']
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '--target', 'zebrad-release', '-t', 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA', '.']

-images: ['gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA']
+images:
+- 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/tests/$BRANCH_NAME:$SHORT_SHA'
+- 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA'

 options:
    machineType: 'N1_HIGHCPU_32'