From 68be226c9d7801ee90b2fdb601dfa42a2dddca42 Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 26 Aug 2021 12:09:03 +0200 Subject: [PATCH] Containerize protobuf generation and remove node build dep The Go dependency is still required to build the pack binary. Use "tilt docker" to use Minikube's Docker instance, if available, removing the local Docker dependency for Minikube users. The Makefile continues to not require Docker and runs buf locally. Remove broken Powershell scripts (can't test on Windows). These scripts should now be substantially easier to write. Change-Id: Ie80bf68e0e468a747861bea36fa5b353d9de110d --- DEVELOP.md | 7 +++--- Dockerfile.proto | 47 +++++++++++++++++++++++++++++++++++++++++ Makefile | 4 +++- Tiltfile | 6 ++++++ generate-protos-all.ps1 | 1 - generate-protos-web.sh | 10 ++------- generate-protos.sh | 9 +------- generate-wasm.ps1 | 8 ------- generate-wasm.sh | 2 +- tools/build.sh | 1 - 10 files changed, 64 insertions(+), 31 deletions(-) create mode 100644 Dockerfile.proto delete mode 100644 generate-protos-all.ps1 delete mode 100644 generate-wasm.ps1 diff --git a/DEVELOP.md b/DEVELOP.md index 16471001..c747c1fb 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -5,11 +5,12 @@ The following dependencies are required for local development: - [Go](https://golang.org/dl/) >= 1.16.6 -- [Docker](https://docs.docker.com/engine/install/) / moby-engine >= 19.03 - [Tilt](http://tilt.dev/) >= 0.20.8 -- [NodeJS/npm](https://nodejs.org/en/download/) >= 14 - Any of the local Kubernetes clusters supported by Tilt. - We recommend [minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/) >= v1.21.0 with the kvm2 driver. + We strongly recommend [minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/) >= + v1.21.0 with the kvm2 driver. + - Tilt will use Minikube's embedded Docker server. If Minikube is not used, a local instance of + [Docker](https://docs.docker.com/engine/install/) / moby-engine >= 19.03 is required. See the [Tilt docs](https://docs.tilt.dev/install.html) docs on how to set up your local cluster - it won't take more than a few minutes to set up! Example minikube invocation, adjust limits as needed: diff --git a/Dockerfile.proto b/Dockerfile.proto new file mode 100644 index 00000000..9885dc93 --- /dev/null +++ b/Dockerfile.proto @@ -0,0 +1,47 @@ +# syntax=docker.io/docker/dockerfile:experimental@sha256:de85b2f3a3e8a2f7fe48e8e84a65f6fdd5cd5183afa6412fff9caa6871649c44 +FROM docker.io/golang:1.17.0@sha256:06e92e576fc7a7067a268d47727f3083c0a564331bfcbfdde633157fc91fb17d AS go-tools + +RUN mkdir /app + +ADD tools/build.sh /app/tools/ +ADD tools/go.* /app/tools/ + +RUN --mount=type=cache,target=/root/.cache --mount=type=cache,target=/go \ + cd /app/tools && CGO_ENABLED=0 ./build.sh + +# syntax=docker.io/docker/dockerfile:experimental@sha256:de85b2f3a3e8a2f7fe48e8e84a65f6fdd5cd5183afa6412fff9caa6871649c44 +FROM docker.io/golang:1.17.0@sha256:06e92e576fc7a7067a268d47727f3083c0a564331bfcbfdde633157fc91fb17d AS go-build + +COPY --from=go-tools /app /app + +ADD buf.* /app +ADD proto /app/proto + +RUN --mount=type=cache,target=/root/.cache \ + cd /app && \ + tools/bin/buf lint && \ + tools/bin/buf generate + +FROM node:16-alpine@sha256:004dbac84fed48e20f9888a23e32fa7cf83c2995e174a78d41d9a9dd1e051a20 AS node-build + +COPY --from=go-tools /app /app + +ADD buf.* /app +ADD proto /app/proto + +ADD tools/package.json /app/tools/ +ADD tools/package-lock.json /app/tools/ + +RUN --mount=type=cache,target=/root/.cache --mount=type=cache,target=/root/.npm \ + cd /app/tools && npm ci + +RUN --mount=type=cache,target=/root/.cache \ + cd /app && \ + tools/bin/buf generate --template buf.gen.web.yaml + +FROM scratch AS go-export +COPY --from=go-build /app/node/pkg/proto pkg/proto + +FROM scratch AS node-export +COPY --from=node-build /app/sdk/js/src/proto sdk/js/src/proto +COPY --from=node-build /app/explorer/src/proto explorer/src/proto diff --git a/Makefile b/Makefile index 8c279504..728286fd 100755 --- a/Makefile +++ b/Makefile @@ -22,7 +22,9 @@ install: .PHONY: generate generate: dirs - ./generate-protos.sh + cd tools && ./build.sh + rm -rf node/pkg/proto + tools/bin/buf generate .PHONY: node node: $(BIN)/guardiand diff --git a/Tiltfile b/Tiltfile index 7ad2f33c..170db023 100644 --- a/Tiltfile +++ b/Tiltfile @@ -210,6 +210,12 @@ build_cloud_function( builder = "gcr.io/buildpacks/builder:v1", ) +local_resource( + name = "pack-bin", + cmd = "go build -mod=readonly -o bin/pack github.com/buildpacks/pack/cmd/pack", + dir = "tools", +) + k8s_yaml_with_ns("devnet/bigtable.yaml") k8s_resource("bigtable-emulator", port_forwards = [ diff --git a/generate-protos-all.ps1 b/generate-protos-all.ps1 deleted file mode 100644 index 4489847a..00000000 --- a/generate-protos-all.ps1 +++ /dev/null @@ -1 +0,0 @@ -docker run -it --rm -v ${pwd}:/wormhole golang sh -c "apt update && apt install -y nodejs npm && cd /wormhole && ./generate-protos.sh && ./generate-protos-web.sh" \ No newline at end of file diff --git a/generate-protos-web.sh b/generate-protos-web.sh index d4189a77..f42e627b 100755 --- a/generate-protos-web.sh +++ b/generate-protos-web.sh @@ -1,11 +1,5 @@ #!/usr/bin/env bash -( - cd tools/ - npm ci -) +rm -rf explorer/src/proto sdk/js/src/proto -rm -rf explorer/src/proto -mkdir -p explorer/src/proto - -tools/bin/buf generate --template buf.gen.web.yaml +DOCKER_BUILDKIT=1 tilt docker build -- --target node-export -f Dockerfile.proto -o type=local,dest=. . diff --git a/generate-protos.sh b/generate-protos.sh index 54cc3048..0b119ac0 100755 --- a/generate-protos.sh +++ b/generate-protos.sh @@ -1,15 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -( - cd tools/ - ./build.sh -) - # TODO(leo): remove after a while rm -rf bridge rm -rf node/pkg/proto - -tools/bin/buf lint -tools/bin/buf generate +DOCKER_BUILDKIT=1 tilt docker build -- --target go-export -f Dockerfile.proto -o type=local,dest=node . diff --git a/generate-wasm.ps1 b/generate-wasm.ps1 deleted file mode 100644 index fe98b7f1..00000000 --- a/generate-wasm.ps1 +++ /dev/null @@ -1,8 +0,0 @@ -cd solana -docker build -t localhost/certusone/wormhole-wasmpack:latest -f Dockerfile.wasm . -docker run --rm -it --workdir /usr/src/bridge/bridge/program -v ${pwd}/../sdk/js/src/solana/core:/usr/src/bridge/bridge/program/pkg -e EMITTER_ADDRESS=11111111111111111111111111111115 -e BRIDGE_ADDRESS=11111111111111111111111111111115 localhost/certusone/wormhole-wasmpack:latest /usr/local/cargo/bin/wasm-pack build --target bundler -- --features wasm -docker run --rm -it --workdir /usr/src/bridge/modules/token_bridge/program -v ${pwd}/../sdk/js/src/solana/token:/usr/src/bridge/modules/token_bridge/program/pkg -e EMITTER_ADDRESS=11111111111111111111111111111115 -e BRIDGE_ADDRESS=11111111111111111111111111111115 localhost/certusone/wormhole-wasmpack:latest /usr/local/cargo/bin/wasm-pack build --target bundler -- --features wasm -docker run --rm -it --workdir /usr/src/bridge/bridge/program -v ${pwd}/../clients/solana/pkg:/usr/src/bridge/bridge/program/pkg -e EMITTER_ADDRESS=11111111111111111111111111111115 -e BRIDGE_ADDRESS=11111111111111111111111111111115 localhost/certusone/wormhole-wasmpack:latest /usr/local/cargo/bin/wasm-pack build --target nodejs -- --features wasm -cp ${pwd}/../clients/solana/pkg/* ${pwd}/../clients/token_bridge/pkg/core -docker run --rm -it --workdir /usr/src/bridge/modules/token_bridge/program -v ${pwd}/../clients/token_bridge/pkg/token:/usr/src/bridge/modules/token_bridge/program/pkg -e EMITTER_ADDRESS=11111111111111111111111111111115 -e BRIDGE_ADDRESS=11111111111111111111111111111115 localhost/certusone/wormhole-wasmpack:latest /usr/local/cargo/bin/wasm-pack build --target nodejs -- --features wasm -cd .. \ No newline at end of file diff --git a/generate-wasm.sh b/generate-wasm.sh index 54d1a784..e9d329c8 100755 --- a/generate-wasm.sh +++ b/generate-wasm.sh @@ -4,5 +4,5 @@ set -euo pipefail ( cd solana - DOCKER_BUILDKIT=1 docker build -f Dockerfile.wasm -o type=local,dest=.. . + DOCKER_BUILDKIT=1 tilt docker build -- -f Dockerfile.wasm -o type=local,dest=.. . ) diff --git a/tools/build.sh b/tools/build.sh index 20a9514f..3b65890b 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -7,4 +7,3 @@ go build -mod=readonly -o bin/protoc-gen-openapiv2 github.com/grpc-ecosystem/grp go build -mod=readonly -o bin/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc go build -mod=readonly -o bin/buf github.com/bufbuild/buf/cmd/buf go build -mod=readonly -o bin/cobra github.com/spf13/cobra/cobra -go build -mod=readonly -o bin/pack github.com/buildpacks/pack/cmd/pack