feat(lightwalletd): add build and CI pipeline (#3657)

* add(actions): lightwalletd continous integrations

* refactor(actions): build lightwalletd and reuse it in zebra

- Download lightwalletd source code
- Create a new Dockerfile for lightwalletd
- Use lightwalletd binary in Zebra's image
- Create a specific step to build/update lightwalletd
- Add lightwalletd integration test to the test suite
- Remove lightwalletd.yml, as it was harder to control

* fix(build): remove extra port being exposed

* fix(lightwalletd): test should be after `--` in cargo test

* revert(lint): do not lint external code as it can be confusing

* fix(test): lightwalletd_integration test is not ignored

* docs(docker): clarify the addition of unused args

* refactor(docker): organize Dockerfiles and remove unused

Fixes: #3344

* fix(actions): activate workflows on correct path changes

* test

* revert previous commit

* feat(build): add arm64 support with cross-compilation (#3659)

* feat(build): add arrm64 support

* fix(build): do not install google-compute-engine in arm64

This package is not available for this platform

* fix(build): do not build arm64 for tests

* fix(changes): reduce changelog

* Revert "feat(build): add arm64 support with cross-compilation (#3659)"

This reverts commit 291e00c405.

.github/workflows/cd.yml

@@ -67,7 +67,7 @@ jobs:
       with:
         target: runtime
         context: .
-        file: ./docker/Dockerfile.build
+        file: ./docker/Dockerfile
         tags: |
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}:latest
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}:${{ env.GITHUB_SHA_SHORT }}

.github/workflows/ci.patch.yml

@@ -8,7 +8,6 @@ on:
       - '**/Cargo.toml'
       - '**/Cargo.lock'
       - '**/deny.toml'
-      - 'docker/**'
       - '.github/workflows/ci.yml'
 
 jobs:

.github/workflows/test.yml

@@ -72,7 +72,7 @@ jobs:
         with:
           target: tester
           context: .
-          file: ./docker/Dockerfile.build
+          file: ./docker/Dockerfile
           tags: |
             ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
             ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
@@ -146,6 +146,26 @@ jobs:
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
           docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
 
+  test-lightwalletd-integration:
+    name: Test integration with lightwalletd
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Run tests with included lightwalletd binary
+        run: |
+          docker pull ${{ env.GAR_BASE }}/zebrad-test:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS -e ZEBRA_TEST_LIGHTWALLETD --name zebrad-tests -t ${{ env.GAR_BASE }}/zebrad-test:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance -- lightwalletd_integration 
+        env:
+          ZEBRA_TEST_LIGHTWALLETD: '1'
+
   regenerate-stateful-disks:
     name: Regenerate stateful disks
     runs-on: ubuntu-latest

.github/workflows/zcash-lightwalletd.yml

@@ -0,0 +1,73 @@
+name: zcash-lightwalletd
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'zebra-rpc/**'
+      - 'zebrad/tests/acceptance.rs'
+      - 'zebrad/src/config.rs'
+      - 'zebrad/src/commands/start.rs'
+      - 'docker/zcash-lightwalletd/Dockerfile'
+      - '.github/workflows/zcash-lightwalletd.yml'
+
+env:
+  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+  GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
+  GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
+  IMAGE_NAME: lightwalletd
+
+jobs:
+  build:
+    name: Build images
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        repository: adityapk00/lightwalletd
+        ref: 'master'
+        persist-credentials: false
+
+    - uses: actions/checkout@v2.4.0
+      with:
+        path: zebra
+        persist-credentials: false
+
+    - name: Inject slug/short variables
+      uses: rlespinasse/github-slug-action@v4
+
+    # Setup gcloud CLI
+    - name: Authenticate to Google Cloud
+      id: auth
+      uses: google-github-actions/auth@v0.6.0
+      with:
+        credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+    # Setup Docker Buildx to allow use of docker cache layers from GH
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to Google Artifact Registry
+      uses: docker/login-action@v1.12.0
+      with:
+        registry: us-docker.pkg.dev
+        username: _json_key
+        password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+    # Build and push image to Google Artifact Registry
+    - name: Build & push
+      id: docker_build
+      uses: docker/build-push-action@v2.9.0
+      with:
+        target: builder
+        context: .
+        file: ./zebra/docker/zcash-lightwalletd/Dockerfile
+        tags: |
+          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
+          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+        push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max

.github/workflows/zcash-params.yml

@@ -9,6 +9,7 @@ on:
       - 'zebra-consensus/src/primitives/groth16/params.rs'
       - 'zebra-consensus/src/chain.rs'
       - 'zebrad/src/commands/start.rs'
+      - 'docker/zcash-params/Dockerfile'
       - '.github/workflows/zcash-params.yml'
 
 env:
@@ -57,7 +58,7 @@ jobs:
       with:
         target: builder
         context: .
-        file: ./docker/Dockerfile.params
+        file: ./docker/zcash-params/Dockerfile
         tags: |
           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}

docker/Dockerfile

@@ -56,6 +56,7 @@ FROM builder AS tester
 # Pre-download Zcash Sprout and Sapling parameters
 # TODO: do not hardcode the user /root/ even though is a safe assumption
 COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/zcash-params /root/.zcash-params /root/.zcash-params
+COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /lightwalletd /usr/local/bin
 
 # Skip IPv6 tests by default, as some CI environment don't have IPv6 available
 ARG ZEBRA_SKIP_IPV6_TESTS=1

docker/Dockerfile.test

@@ -1,30 +0,0 @@
-FROM rust:buster
-
-RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-	make cmake g++ gcc llvm libclang-dev clang
-
-RUN mkdir /zebra
-WORKDIR /zebra
-
-ARG SHORT_SHA
-ENV SHORT_SHA $SHORT_SHA
-
-ENV RUST_BACKTRACE full
-ENV CARGO_HOME /zebra/.cargo/
-# Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
-# GitHub Actions time limit (6 hours), so we needed to make it faster.
-ENV RUSTFLAGS -O
-
-RUN rustc -V; cargo -V; rustup -V
-
-EXPOSE 8233 18233
-
-COPY . .
-
-# Pre-download Zcash Sprout and Sapling parameters
-RUN cargo run --verbose --bin zebrad download
-
-RUN cargo test --all --no-run
-
-CMD cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored

docker/zcash-lightwalletd/Dockerfile

@@ -0,0 +1,61 @@
+ARG ZCASHD_CONF_PATH=/etc/zcash.conf
+ARG LWD_GRPC_PORT=9067
+ARG LWD_HTTP_PORT=9068
+
+##
+## Build
+##
+FROM golang:1.17 AS build
+
+# Create and change to the app directory.
+WORKDIR /app
+
+# Retrieve application dependencies.
+# This allows the container build to reuse cached dependencies.
+# Expecting to copy go.mod and if present go.sum.
+COPY go.mod ./
+COPY go.sum ./
+
+# Do not use `go get` as it updates the requirements listed in your go.mod file.
+# `go mod download` does not add new requirements or update existing requirements.
+RUN go mod download
+
+# Copy local code to the container image.
+COPY . ./
+
+# Build and install the binary.
+RUN go build -v -o /lightwalletd
+
+ARG ZCASHD_CONF_PATH
+
+RUN set -ex; \
+  { \
+    echo "rpcuser=zcashrpc"; \
+    echo "rpcpassword=`head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo ''`" \
+    echo "rpcbind=127.0.0.1"; \
+    echo "rpcport=8232"; \
+  } > "${ZCASHD_CONF_PATH}"
+
+ENTRYPOINT ["/lightwalletd"]
+CMD ["--no-tls-very-insecure", "--grpc-bind-addr=0.0.0.0:9067",  "--http-bind-addr=0.0.0.0:9068", "--log-file=/dev/stdout", "--log-level=7"]
+
+##
+## Deploy
+##
+FROM debian:bullseye-slim as runtime
+
+ARG ZCASHD_CONF_PATH
+# Maintain backward compatibility with mainstream repo using this ARGs in docker-compose
+ARG LWD_GRPC_PORT
+ARG LWD_HTTP_PORT
+
+WORKDIR /
+
+COPY --from=build /lightwalletd /usr/local/bin
+COPY --from=build $ZCASHD_CONF_PATH ./
+
+EXPOSE 9067
+EXPOSE 9068
+
+ENTRYPOINT ["lightwalletd"]
+CMD ["--no-tls-very-insecure", "--grpc-bind-addr=0.0.0.0:9067",  "--http-bind-addr=0.0.0.0:9068", "--log-file=/dev/stdout", "--log-level=7"]