From aeb673a1642cd89f683b9ee6f46369fcb7668b12 Mon Sep 17 00:00:00 2001 From: Leo Date: Sun, 22 Aug 2021 21:52:48 +0200 Subject: [PATCH] Add presubmit CI test that runs "tilt ci" - Use "npm ci" instead of "npm install" in a couple of places. - Migrate generate-wasm.sh to a Dockerfile so it can be cached. - Disable namespace creation and explorer build in CI. Change-Id: I2e6ee806438863ca81ada701e14684f9f4cc0a76 --- Dockerfile.client | 21 ++++++----- Tiltfile | 60 ++++++++++++++++++------------- bridge/cmd/guardiand/publicweb.go | 1 - devnet/terra-devnet.yaml | 2 +- generate-wasm.sh | 29 +-------------- jenkins-presubmit.groovy | 24 ++++++++++++- solana/Dockerfile.wasm | 32 +++++++++++++++-- 7 files changed, 100 insertions(+), 69 deletions(-) diff --git a/Dockerfile.client b/Dockerfile.client index 22c7ba61b..3c7904ad8 100644 --- a/Dockerfile.client +++ b/Dockerfile.client @@ -13,19 +13,18 @@ RUN --mount=type=cache,target=/root/.cache \ ADD ethereum /usr/src/ethereum WORKDIR /usr/src/ethereum -RUN --mount=type=cache,target=/usr/src/ethereum/node_modules \ - npm install +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=/root/.npm \ + npm ci ADD clients/token_bridge /usr/src/clients/token_bridge WORKDIR /usr/src/clients/token_bridge -RUN --mount=type=cache,target=/usr/src/ethereum/node_modules \ - --mount=type=cache,target=/usr/src/clients/token_bridge/node_modules \ - npm install && \ - npm run build-contracts && \ - npm run build && \ - cp -r node_modules node_modules_cached - -RUN rm -rf node_modules && mv node_modules_cached node_modules # Make cached node_modules part of the image +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=/root/.npm \ + set -xe && \ + npm ci && \ + npm run build-contracts && \ + npm run build ADD solana /usr/src/solana ADD proto /usr/src/proto @@ -35,9 +34,9 @@ ENV EMITTER_ADDRESS="11111111111111111111111111111115" ENV BRIDGE_ADDRESS="Bridge1p5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o" RUN --mount=type=cache,target=/root/.cache \ - --mount=type=cache,target=target \ --mount=type=cache,target=bridge/target \ --mount=type=cache,target=modules/token_bridge/target \ + set -xe && \ cargo build --manifest-path ./bridge/Cargo.toml --package client --release && \ cargo build --manifest-path ./modules/token_bridge/Cargo.toml --package client --release && \ cp /usr/local/cargo/bin/solana /usr/local/bin && \ diff --git a/Tiltfile b/Tiltfile index 73d9ccb2d..bedeb14f1 100644 --- a/Tiltfile +++ b/Tiltfile @@ -6,9 +6,15 @@ # load("ext://namespace", "namespace_create", "namespace_inject") -load('ext://secret', 'secret_yaml_generic') +load("ext://secret", "secret_yaml_generic") + +allow_k8s_contexts("ci") + +# Disable telemetry by default +analytics_settings(False) # Runtime configuration +config.define_bool("ci", False, "We are running in CI") config.define_string("num", False, "Number of guardian nodes to run") @@ -30,10 +36,12 @@ namespace = cfg.get("namespace", "wormhole") bigTablePersistence = cfg.get("bigTablePersistence", False) gcpProject = cfg.get("gcpProject", None) bigTableKeyPath = cfg.get("bigTableKeyPath", "./bigtable-writer.json") +ci = cfg.get("ci", False) # namespace -namespace_create(namespace) +if not ci: + namespace_create(namespace) def k8s_yaml_with_ns(objects): return k8s_yaml(namespace_inject(objects, namespace)) @@ -61,8 +69,8 @@ if bigTablePersistence: k8s_yaml_with_ns( secret_yaml_generic( "bridge-bigtable-key", - from_file = "bigtable-key.json=" + bigTableKeyPath - ) + from_file = "bigtable-key.json=" + bigTableKeyPath, + ), ) docker_build( @@ -85,7 +93,7 @@ def build_bridge_yaml(): container["command"] += [ "--bigTablePersistenceEnabled", "--bigTableGCPProject", - gcpProject + gcpProject, ] return encode_yaml_stream(bridge_yaml) @@ -101,7 +109,7 @@ k8s_resource("guardian", resource_deps = ["proto-gen", "solana-devnet"], port_fo docker_build( ref = "pyth", context = ".", - dockerfile = "third_party/pyth/Dockerfile" + dockerfile = "third_party/pyth/Dockerfile", ) k8s_yaml_with_ns("./devnet/pyth.yaml") @@ -114,7 +122,7 @@ k8s_yaml_with_ns("./devnet/envoy-proxy.yaml") k8s_resource( "envoy-proxy", resource_deps = ["guardian"], - objects = ["envoy-proxy:ConfigMap:wormhole"], + objects = ["envoy-proxy:ConfigMap"], port_forwards = [ port_forward(8080, name = "gRPC proxy for guardian's publicRPC data [:8080]"), port_forward(9901, name = "gRPC proxy admin [:9901]"), # for proxy debugging @@ -179,26 +187,28 @@ k8s_resource("eth-devnet", port_forwards = [ # explorer web app -docker_build( - ref = "explorer", - context = "./explorer", - dockerfile = "./explorer/Dockerfile", - ignore = ["./explorer/node_modules"], - live_update = [ - sync("./explorer/src", "/home/node/app/src"), - sync("./explorer/public", "/home/node/app/public"), - ], -) +# TOOD: the explorer web app does not currently build +if not ci: + docker_build( + ref = "explorer", + context = "./explorer", + dockerfile = "./explorer/Dockerfile", + ignore = ["./explorer/node_modules"], + live_update = [ + sync("./explorer/src", "/home/node/app/src"), + sync("./explorer/public", "/home/node/app/public"), + ], + ) -k8s_yaml_with_ns("devnet/explorer.yaml") + k8s_yaml_with_ns("devnet/explorer.yaml") -k8s_resource( - "explorer", - resource_deps = ["envoy-proxy", "proto-gen-web"], - port_forwards = [ - port_forward(8001, name = "Explorer Web UI [:8001]"), - ], -) + k8s_resource( + "explorer", + resource_deps = ["envoy-proxy", "proto-gen-web"], + port_forwards = [ + port_forward(8001, name = "Explorer Web UI [:8001]"), + ], + ) # terra devnet diff --git a/bridge/cmd/guardiand/publicweb.go b/bridge/cmd/guardiand/publicweb.go index d86bc6f98..2a3802388 100644 --- a/bridge/cmd/guardiand/publicweb.go +++ b/bridge/cmd/guardiand/publicweb.go @@ -11,7 +11,6 @@ import ( "golang.org/x/crypto/acme" "golang.org/x/crypto/acme/autocert" "google.golang.org/grpc" - "log" "net" "net/http" "strings" diff --git a/devnet/terra-devnet.yaml b/devnet/terra-devnet.yaml index 0df68a591..8821def05 100644 --- a/devnet/terra-devnet.yaml +++ b/devnet/terra-devnet.yaml @@ -200,4 +200,4 @@ spec: - name: TYPEORM_ENTITIES value: "src/orm/*Entity.ts" restartPolicy: Always - serviceName: terra-fcd \ No newline at end of file + serviceName: terra-fcd diff --git a/generate-wasm.sh b/generate-wasm.sh index 75dff8003..54d1a784b 100755 --- a/generate-wasm.sh +++ b/generate-wasm.sh @@ -4,32 +4,5 @@ set -euo pipefail ( cd solana - mkdir -p ../sdk/js/src/solana/core - mkdir -p ../sdk/js/src/solana/token - docker build -t localhost/certusone/wormhole-wasmpack:latest -f Dockerfile.wasm . - docker run --rm -it --workdir /usr/src/bridge/bridge/program \ - -v $(pwd)/../sdk/js/src/solana/core:/usr/src/bridge/bridge/program/pkg \ - -e EMITTER_ADDRESS=11111111111111111111111111111115 \ - -e BRIDGE_ADDRESS=11111111111111111111111111111115 \ - localhost/certusone/wormhole-wasmpack:latest \ - /usr/local/cargo/bin/wasm-pack build --target bundler -- --features wasm - docker run --rm -it --workdir /usr/src/bridge/modules/token_bridge/program \ - -v $(pwd)/../sdk/js/src/solana/token:/usr/src/bridge/modules/token_bridge/program/pkg \ - -e EMITTER_ADDRESS=11111111111111111111111111111115 \ - -e BRIDGE_ADDRESS=11111111111111111111111111111115 \ - localhost/certusone/wormhole-wasmpack:latest \ - /usr/local/cargo/bin/wasm-pack build --target bundler -- --features wasm - docker run --rm -it --workdir /usr/src/bridge/bridge/program \ - -v $(pwd)/../clients/solana/pkg:/usr/src/bridge/bridge/program/pkg \ - -e EMITTER_ADDRESS=11111111111111111111111111111115 \ - -e BRIDGE_ADDRESS=11111111111111111111111111111115 \ - localhost/certusone/wormhole-wasmpack:latest \ - /usr/local/cargo/bin/wasm-pack build --target nodejs -- --features wasm - cp $(pwd)/../clients/solana/pkg/. $(pwd)/../clients/token_bridge/pkg/core -R - docker run --rm -it --workdir /usr/src/bridge/modules/token_bridge/program \ - -v $(pwd)/../clients/token_bridge/pkg/token:/usr/src/bridge/modules/token_bridge/program/pkg \ - -e EMITTER_ADDRESS=11111111111111111111111111111115 \ - -e BRIDGE_ADDRESS=11111111111111111111111111111115 \ - localhost/certusone/wormhole-wasmpack:latest \ - /usr/local/cargo/bin/wasm-pack build --target nodejs -- --features wasm + DOCKER_BUILDKIT=1 docker build -f Dockerfile.wasm -o type=local,dest=.. . ) diff --git a/jenkins-presubmit.groovy b/jenkins-presubmit.groovy index 6e8c8dab6..316b38c32 100644 --- a/jenkins-presubmit.groovy +++ b/jenkins-presubmit.groovy @@ -1,3 +1,5 @@ +final kubeCleanup = "kubectl delete --namespace=\$DEPLOY_NS service,statefulset,configmap,pod --all" + pipeline { agent none stages { @@ -12,15 +14,35 @@ pipeline { } steps { gerritCheck checks: ['jenkins:test': 'RUNNING'], message: "Running on ${env.NODE_NAME}" - sh "git show HEAD" + + echo "Gerrit change: ${GERRIT_CHANGE_URL}" + echo "Tilt progress dashboard: https://${DASHBOARD_URL}" + + sh """ + kubectl config set-context ci --namespace=$DEPLOY_NS + kubectl config use-context ci + """ + + sh kubeCleanup + + sh "./generate-wasm.sh" + + timeout(time: 60, unit: 'MINUTES') { + sh "tilt ci -- --ci --namespace=$DEPLOY_NS --num=1" + } } post { success { + gerritReview labels: [Verified: 1] gerritCheck checks: ['jenkins:test': 'SUCCESSFUL'] } unsuccessful { + gerritReview labels: [Verified: -1] gerritCheck checks: ['jenkins:test': 'FAILED'] } + cleanup { + sh kubeCleanup + } } } } diff --git a/solana/Dockerfile.wasm b/solana/Dockerfile.wasm index 7d3ed36ad..5f467a4c9 100644 --- a/solana/Dockerfile.wasm +++ b/solana/Dockerfile.wasm @@ -1,5 +1,5 @@ # syntax=docker.io/docker/dockerfile:experimental@sha256:de85b2f3a3e8a2f7fe48e8e84a65f6fdd5cd5183afa6412fff9caa6871649c44 -FROM docker.io/library/rust:1.49@sha256:a50165ea96983c21832578afb1c8c028674c965bc1ed43b607871b1f362e06a5 +FROM docker.io/library/rust:1.49@sha256:a50165ea96983c21832578afb1c8c028674c965bc1ed43b607871b1f362e06a5 AS build RUN apt-get update && apt-get install -y libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang RUN rustup component add rustfmt @@ -11,7 +11,35 @@ RUN cargo install wasm-pack ENV RUST_LOG="solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=trace,solana_bpf_loader=debug,solana_rbpf=debug" ENV EMITTER_ADDRESS="11111111111111111111111111111115" +ENV BRIDGE_ADDRESS="Bridge1p5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o" COPY bridge bridge COPY modules modules -COPY solitaire solitaire \ No newline at end of file +COPY solitaire solitaire + +# Compile Wormhole +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=bridge/target \ + cd bridge/program && /usr/local/cargo/bin/wasm-pack build --target bundler -d bundler -- --features wasm + +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=bridge/target \ + cd bridge/program && /usr/local/cargo/bin/wasm-pack build --target nodejs -d nodejs -- --features wasm + +# Compile Token Bridge +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=modules/token_bridge/target \ + cd modules/token_bridge/program && /usr/local/cargo/bin/wasm-pack build --target bundler -d bundler -- --features wasm + +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=modules/token_bridge/target \ + cd modules/token_bridge/program && /usr/local/cargo/bin/wasm-pack build --target nodejs -d nodejs -- --features wasm + +FROM scratch AS export + +COPY --from=build /usr/src/bridge/bridge/program/bundler sdk/js/src/solana/core +COPY --from=build /usr/src/bridge/modules/token_bridge/program/bundler sdk/js/src/solana/token + +COPY --from=build /usr/src/bridge/bridge/program/nodejs clients/solana/pkg +COPY --from=build /usr/src/bridge/bridge/program/nodejs clients/token_bridge/pkg/core +COPY --from=build /usr/src/bridge/modules/token_bridge/program/nodejs clients/token_bridge/pkg/token