Split up big test job into its own workflow

2020-11-19 15:21:03 -05:00 · 2020-11-19 15:21:03 -05:00 · 44970af929
parent 2445d23dd8
commit 44970af929
5 changed files with 82 additions and 130 deletions
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@ -1,13 +1,15 @@
 name: CD

 on:
+  workflow_dispatch:
  push:
    branches:
      - main
-      - cloudbuild

 env:
  PROJECT_ID: zealous-zebra
+  ZONE: us-east1-b
+

 jobs:

@ -43,68 +45,6 @@ jobs:
          --config cloudbuild.yaml \
          --substitutions SHORT_SHA="$SHORT_SHA",BRANCH_NAME="$BRANCH_NAME"

-  test:
-    name: Run integration tests
-    needs: build
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set project and image names
-        run: |
-          BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
-          BRANCH_NAME=${BRANCH_NAME,,} && \
-          REPOSITORY=${GITHUB_REPOSITORY,,} && \
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
-          echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
-          echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
-
-      # Setup gcloud CLI
-      - name: Set up gcloud SDK environment
-        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
-        with:
-          version: '295.0.0'
-          project_id: ${{ env.PROJECT_ID }}
-          service_account_key: ${{ secrets.GCLOUD_AUTH }}
-
-      # Run once: create firewall rule to allow incoming traffic to the nodes
-      # - name: Create Zcash incoming traffic firewall rule
-      #   run: |
-      #     gcloud compute firewall-rules create "allow-zcash" \
-      #     --target-tags zebrad \
-      #     --allow tcp:8233,tcp:18233 \
-      #     --source-ranges 0.0.0.0/0 \
-      #     --description="Allow incoming Zcash traffic from anywhere" \
-
-      # Creates Compute Engine virtual machine instance w/ test container and disks
-      - name: Create instance with test container image
-        run: |
-          gcloud compute instances create-with-container "zebra-tests-$BRANCH_NAME-$SHORT_SHA" \
-          --container-command='/zebra/target/debug/deps/acceptance' \
-          --container-arg='-Z' --container-arg='unstable-options' --container-arg='--include-ignored' \
-          --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/tests/$BRANCH_NAME:$SHORT_SHA" \
-          --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-$SHORT_SHA \
-          --container-restart-policy='never' \
-          --create-disk name=zebrad-cache-$SHORT_SHA,image=zebrad-cache-mainnet-height-1018849 \
-          --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
-          --scopes cloud-platform \
-          --tags zebrad \
-          --zone us-central1-a
-
-      # Wait for acceptance test container is up and stream its results
-      - name: Run acceptance tests
-        run: |
-          gcloud compute ssh "zebra-tests-$BRANCH_NAME-$SHORT_SHA" --zone us-central1-a \
-          --command "echo 'Pulling container image...'; until [[ -n \$(docker ps --all -q --filter name=zebra-test) ]]; do sleep 0.1; done; docker attach \$(docker ps -q -a --filter name=zebra-test)"
-
-      # Clean up
-      - name: Delete test instance
-        # Always run even if the earlier step fails
-        if: ${{ always() }}
-        run: |
-          gcloud compute instances delete "zebra-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone us-central1-a
-
  deploy:
    name: Deploy mainnet nodes
    needs: build
@ -146,7 +86,7 @@ jobs:
          --container-image "gcr.io/$PROJECT_ID/$REPOSITORY/$BRANCH_NAME:$SHORT_SHA" \
          --create-disk name=zebrad-cache-$SHORT_SHA,size=100GB,type=pd-balanced,auto-delete=no \
          --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-$SHORT_SHA \
-          --machine-type n2-standard-4 \
+          --machine-type n2d-standard-4 \
          --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
          --scopes cloud-platform \
          --tags zebrad \
@ -176,7 +116,7 @@ jobs:
          --template "zebrad-$BRANCH_NAME-$SHORT_SHA" \
          --health-check zebrad-tracing-filter \
          --initial-delay 30 \
-          --region us-central1 \
+          --region "$ZONE" \
          --size 2

      # Rolls out update to existing group using the new instance template
@ -186,4 +126,4 @@ jobs:
          gcloud compute instance-groups managed rolling-action start-update \
          "zebrad-$BRANCH_NAME" \
          --version template="zebrad-$BRANCH_NAME-$SHORT_SHA" \
-          --region us-central1 \
+          --region "$ZONE"
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -0,0 +1,73 @@
+name: Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+env:
+  PROJECT_ID: zealous-zebra
+  ZONE: europe-west1-b
+
+jobs:
+
+  test:
+    name: Run all tests
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set project and image names
+        run: |
+          BRANCH_NAME=$(expr $GITHUB_REF : '.*/\(.*\)') && \
+          BRANCH_NAME=${BRANCH_NAME,,} && \
+          REPOSITORY=${GITHUB_REPOSITORY,,} && \
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \
+          echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV && \
+          echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV
+
+      # Setup gcloud CLI
+      - name: Set up gcloud
+        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          version: '295.0.0'
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+      # Run once: create firewall rule to allow incoming traffic to the nodes
+      # - name: Create Zcash incoming traffic firewall rule
+      #   run: |
+      #     gcloud compute firewall-rules create "allow-zcash" \
+      #     --target-tags zebrad \
+      #     --allow tcp:8233,tcp:18233 \
+      #     --source-ranges 0.0.0.0/0 \
+      #     --description="Allow incoming Zcash traffic from anywhere" \
+
+      # Creates Compute Engine virtual machine instance w/ disks
+      - name: Create instance
+        run: |
+          gcloud compute instances create "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" \
+          --boot-disk-size 100GB \
+          --boot-disk-type pd-ssd \
+          --image-family cos-stable \
+          --machine-type n2d-standard-16 \
+          --service-account cos-vm@zealous-zebra.iam.gserviceaccount.com \
+          --scopes cloud-platform \
+          --tags zebrad \
+          --zone "$ZONE"
+
+      # Build and run test container
+      - name: Run all tests
+        run: |
+          gcloud compute ssh "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --zone "$ZONE" \
+          --command "git clone -b test https://github.com/ZcashFoundation/zebra.git; cd zebra/; docker build -f docker/Dockerfile.test -t zebrad-test .; docker run -i zebrad-test"
+
+      # Clean up
+      - name: Delete test instance
+        # Always run even if the earlier step fails
+        if: ${{ always() }}
+        run: |
+          gcloud compute instances delete "zebrad-tests-$BRANCH_NAME-$SHORT_SHA" --delete-disks all --zone "$ZONE"
--- a/53
+++ b/53
@ -1,53 +0,0 @@
-# Builder image
-FROM rust:buster as builder
-
-RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-	make cmake g++ gcc llvm libclang-dev clang
-
-RUN mkdir /zebra
-WORKDIR /zebra
-
-ENV RUST_BACKTRACE 1
-ENV CARGO_HOME /zebra/.cargo/
-
-# Copy local code to the container image.
-# Assumes that we are in the git repo.
-COPY . .
-RUN cargo fetch --verbose
-COPY . .
-RUN rustc -V; cargo -V; rustup -V; cargo test --all && cargo build --release
-RUN find /zebra/target/debug/deps -type f -perm 755 ! -name '*.dylib' ! -name '*.so' | sed -e 'p;s/-.*//' | xargs -n2 mv
-
-
-# Test binaries image
-FROM debian:buster AS zebra-tests
-
-RUN mkdir /zebra
-WORKDIR /zebra
-COPY --from=builder /zebra/target/debug/zebrad /zebra/target/debug/zebrad
-COPY --from=builder /zebra/target/debug/deps/[a-z0-9_]* /zebra/target/debug/deps/
-EXPOSE 8233 18233
-
-
-# Runner image
-FROM debian:buster-slim AS zebrad-release
-
-COPY --from=builder /zebra/target/release/zebrad /
-
-ARG CHECKPOINT_SYNC=true
-ARG NETWORK=Mainnet
-
-RUN printf "[consensus]\n" >> /zebrad.toml
-RUN printf "checkpoint_sync = ${CHECKPOINT_SYNC}\n" >> /zebrad.toml
-RUN printf "[network]\n" >> /zebrad.toml
-RUN printf "network = '${NETWORK}'\n" >> /zebrad.toml
-RUN printf "[state]\n" >> /zebrad.toml
-RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
-RUN printf "memory_cache_bytes = 52428800\n" >> /zebrad.toml
-RUN printf "[tracing]\n" >> /zebrad.toml
-RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml
-
-EXPOSE 3000 8233 18233
-
-CMD [ "/zebrad", "-c", "/zebrad.toml", "start" ]
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@ -1,11 +1,4 @@
 steps:
- name: 'gcr.io/cloud-builders/docker'
-  args: ['build',
-         '--target',
-         'zebra-tests',
-         '-t',
-         'gcr.io/$PROJECT_ID/zcashfoundation/zebra/tests/$BRANCH_NAME:$SHORT_SHA',
-         '.']
 - name: 'gcr.io/cloud-builders/docker'
  args: ['build',
         '--build-arg',
@ -16,10 +9,11 @@ steps:
         'zebrad-release',
         '-t',
         'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA',
+         '-f',
+         'docker/Dockerfile.build',
         '.']

 images:
- 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/tests/$BRANCH_NAME:$SHORT_SHA'
 - 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA'

 options:
--- a/docker/Dockerfile.test
+++ b/docker/Dockerfile.test
@ -15,8 +15,6 @@ RUN rustc -V; cargo -V; rustup -V
 EXPOSE 8233 18233

 COPY . .
-RUN cargo test --all --no-run
-RUN find /zebra/target/debug/deps -type f -perm 755 ! -name '*.dylib' ! -name '*.so' | sed -e 'p;s/-.*//' | xargs -n2 mv

 # Filtering to only run integration tests requires nightly for now
-CMD cargo +nightly test --test '*' --no-fail-fast -- -Zunstable-options --include-ignored
+CMD cargo +nightly test --all --test '*' --no-fail-fast -- -Zunstable-options --include-ignored