ci: merge docker images (#35078)

* ci: add new ci docker image

* ci: use the new docker image

* fix shellcheck

* add readme for the new docker image

* remove old docker images

* remove unused check in docs/build.sh

* use the new image in net.sh

.buildkite/scripts/build-bench.sh

@@ -22,5 +22,5 @@ EOF
 
 # shellcheck disable=SC2016
 group "bench" \
-  "$(build_steps "bench-part-1" ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/bench/part1.sh")" \
-  "$(build_steps "bench-part-2" ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/bench/part2.sh")"
+  "$(build_steps "bench-part-1" "ci/docker-run-default-image.sh ci/bench/part1.sh")" \
+  "$(build_steps "bench-part-2" "ci/docker-run-default-image.sh ci/bench/part2.sh")"

.buildkite/scripts/build-stable.sh

@@ -12,7 +12,7 @@ partitions=$(
   cat <<EOF
 {
   "name": "partitions",
-  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-partition.sh",
+  "command": "ci/docker-run-default-image.sh ci/stable/run-partition.sh",
   "timeout_in_minutes": 30,
   "agent": "$agent",
   "parallelism": 2,
@@ -25,7 +25,7 @@ local_cluster_partitions=$(
   cat <<EOF
 {
   "name": "local-cluster",
-  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-local-cluster-partially.sh",
+  "command": "ci/docker-run-default-image.sh ci/stable/run-local-cluster-partially.sh",
   "timeout_in_minutes": 30,
   "agent": "$agent",
   "parallelism": 5,
@@ -38,7 +38,7 @@ localnet=$(
   cat <<EOF
 {
   "name": "localnet",
-  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-localnet.sh",
+  "command": "ci/docker-run-default-image.sh ci/stable/run-localnet.sh",
   "timeout_in_minutes": 30,
   "agent": "$agent"
 }

ci/buildkite-pipeline.sh

@@ -140,9 +140,9 @@ wait_step() {
 }
 
 all_test_steps() {
-  command_step checks1 ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20 check
-  command_step checks2 ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-dev-context-only-utils.sh check-bins" 15 check
-  command_step checks3 ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-dev-context-only-utils.sh check-all-targets" 15 check
+  command_step checks1 "ci/docker-run-default-image.sh ci/test-checks.sh" 20 check
+  command_step checks2 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-bins" 15 check
+  command_step checks3 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-all-targets" 15 check
   wait_step
 
   # Full test suite
@@ -156,7 +156,7 @@ all_test_steps() {
              ^ci/rust-version.sh \
              ^ci/test-docs.sh \
       ; then
-    command_step doctest ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-docs.sh" 15
+    command_step doctest "ci/docker-run-default-image.sh ci/test-docs.sh" 15
   else
     annotate --style info --context test-docs \
       "Docs skipped as no .rs files were modified"
@@ -182,7 +182,7 @@ all_test_steps() {
              cargo-test-sbf$ \
       ; then
     cat >> "$output_file" <<"EOF"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable-sbf.sh"
+  - command: "ci/docker-run-default-image.sh ci/test-stable-sbf.sh"
     name: "stable-sbf"
     timeout_in_minutes: 35
     artifact_paths: "sbf-dumps.tar.bz2"
@@ -226,7 +226,7 @@ EOF
              ^ci/test-stable.sh \
              ^sdk/ \
       ; then
-    command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20
+    command_step wasm "ci/docker-run-default-image.sh ci/test-wasm.sh" 20
   else
     annotate --style info \
       "wasm skipped as no relevant files were modified"
@@ -258,7 +258,7 @@ EOF
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 80
+    command_step coverage "ci/docker-run-default-image.sh ci/test-coverage.sh" 80
   else
     annotate --style info --context test-coverage \
       "Coverage skipped as no .rs files were modified"
@@ -296,7 +296,7 @@ pull_or_push_steps() {
 
     if [ -z "$diff_other_than_version_bump" ]; then
       echo "Diff only contains version bump."
-      command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20
+      command_step checks "ci/docker-run-default-image.sh ci/test-checks.sh" 20
       exit 0
     fi
   fi

ci/buildkite-secondary.yml

@@ -3,7 +3,7 @@
 # Pull requests to not run these steps.
 steps:
   - name: "cargo audit"
-    command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/do-audit.sh"
+    command: "ci/docker-run-default-image.sh ci/do-audit.sh"
     agents:
       queue: "release-build"
     timeout_in_minutes: 10

ci/buildkite-solana-private.sh

@@ -134,7 +134,7 @@ wait_step() {
 }
 
 all_test_steps() {
-  command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20
+  command_step checks "ci/docker-run-default-image.sh ci/test-checks.sh" 20
   wait_step
 
   # Full test suite
@@ -146,7 +146,7 @@ all_test_steps() {
              ^ci/rust-version.sh \
              ^ci/test-docs.sh \
       ; then
-    command_step doctest ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-docs.sh" 15
+    command_step doctest "ci/docker-run-default-image.sh ci/test-docs.sh" 15
   else
     annotate --style info --context test-docs \
       "Docs skipped as no .rs files were modified"
@@ -168,7 +168,7 @@ all_test_steps() {
              ^sdk/ \
       ; then
     cat >> "$output_file" <<"EOF"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable-sbf.sh"
+  - command: "ci/docker-run-default-image.sh ci/test-stable-sbf.sh"
     name: "stable-sbf"
     timeout_in_minutes: 35
     artifact_paths: "sbf-dumps.tar.bz2"
@@ -208,7 +208,7 @@ EOF
              ^ci/test-stable.sh \
              ^sdk/ \
       ; then
-    command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20
+    command_step wasm "ci/docker-run-default-image.sh ci/test-wasm.sh" 20
   else
     annotate --style info \
       "wasm skipped as no relevant files were modified"
@@ -238,7 +238,7 @@ EOF
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 80
+    command_step coverage "ci/docker-run-default-image.sh ci/test-coverage.sh" 80
   else
     annotate --style info --context test-coverage \
       "Coverage skipped as no .rs files were modified"

ci/dependabot-pr.sh

@@ -11,7 +11,7 @@ fi
 
 source ci/rust-version.sh stable
 
-ci/docker-run.sh $rust_nightly_docker_image ci/dependabot-updater.sh
+ci/docker-run-default-image.sh ci/dependabot-updater.sh
 
 if [[ $(git status --short :**/Cargo.lock | wc -l) -eq 0 ]]; then
   echo --- ok

ci/docker-run-default-image.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1091
+source "$here/rust-version.sh"
+
+"$here/docker-run.sh" "${ci_docker_image:?}" "$@"

ci/docker-rust-nightly/Dockerfile

@@ -1,21 +0,0 @@
-FROM solanalabs/rust:1.75.0
-
-ARG date
-ARG GRCOV_VERSION=v0.8.18
-
-RUN \
-  rustup install nightly-$date && \
-  rustup component add clippy --toolchain=nightly-$date && \
-  rustup component add rustfmt --toolchain=nightly-$date && \
-  rustup show && \
-  rustc --version && \
-  cargo --version && \
-  # grcov
-  curl -LOsS "https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/grcov-x86_64-unknown-linux-musl.tar.bz2" && \
-  tar -xf grcov-x86_64-unknown-linux-musl.tar.bz2 && \
-  mv ./grcov $CARGO_HOME/bin && \
-  rm grcov-x86_64-unknown-linux-musl.tar.bz2 && \
-  # codecov
-  curl -Os https://uploader.codecov.io/latest/linux/codecov && \
-  chmod +x codecov && \
-  mv codecov /usr/bin

ci/docker-rust-nightly/README.md

@@ -1,42 +0,0 @@
-Docker image containing rust nightly and some preinstalled crates used in CI.
-
-This image may be manually updated by running `CI=true ./build.sh` if you are a member
-of the [Solana Labs](https://hub.docker.com/u/solanalabs/) Docker Hub
-organization.
-
-## Moving to a newer nightly
-
-NOTE: Follow instructions in docker-rust/README.md before this when updating the stable
-rust version as well.
-
-We pin the version of nightly (see the `ARG nightly=xyz` line in `Dockerfile`)
-to avoid the build breaking at unexpected times, as occasionally nightly will
-introduce breaking changes.
-
-To update the pinned version:
-1. Edit `Dockerfile` to match the desired stable rust version to base on if needed.
-1. Run `ci/docker-rust-nightly/build.sh` to rebuild the nightly image locally,
-   or potentially `ci/docker-rust-nightly/build.sh YYYY-MM-DD` if there's a
-   specific YYYY-MM-DD that is desired (default is today's build).
-   Check https://rust-lang.github.io/rustup-components-history/ for build
-   status
-1. Update `ci/rust-version.sh` to reflect the new nightly `YYYY-MM-DD`
-1. Run `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-checks.sh`
-   and `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-coverage.sh [args]...`
-   to confirm the new nightly image builds.  Fix any issues as needed
-1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized.
-1. Run `CI=true ci/docker-rust-nightly/build.sh YYYY-MM-DD` to push the new nightly image to dockerhub.com.
-1. Send a PR with the `ci/rust-version.sh` change and any codebase adjustments needed.
-
-## Troubleshooting
-
-### Resource is denied
-
-When running `CI=true ci/docker-rust-nightly/build.sh`, you see:
-
-```
-denied: requested access to the resource is denied
-```
-
-Run `docker login` to enable pushing images to Docker Hub. Contact @mvines or @garious
-to get write access.

ci/docker-rust-nightly/build.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-cd "$(dirname "$0")"
-
-platform=()
-if [[ $(uname -m) = arm64 ]]; then
-  # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr
-  platform+=(--platform linux/amd64)
-fi
-
-nightlyDate=${1:-$(date +%Y-%m-%d)}
-docker build "${platform[@]}" -t solanalabs/rust-nightly:"$nightlyDate" --build-arg date="$nightlyDate" .
-
-maybeEcho=
-if [[ -z $CI ]]; then
-  echo "Not CI, skipping |docker push|"
-  maybeEcho="echo"
-fi
-$maybeEcho docker push solanalabs/rust-nightly:"$nightlyDate"

ci/docker-rust/README.md

@@ -1,11 +0,0 @@
-Docker image containing rust and some preinstalled packages used in CI.
-
-NOTE: Recreate rust-nightly docker image after this when updating the stable rust
-version! Both docker images must be updated in tandem.
-
-This image is manually maintained:
-1. Edit `Dockerfile` to match the desired rust version
-1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized.
-1. Run `./build.sh` to publish the new image, if you are a member of the [Solana
-   Labs](https://hub.docker.com/u/solanalabs/) Docker Hub organization.
-

ci/docker-rust/build.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-cd "$(dirname "$0")"
-
-
-platform=()
-if [[ $(uname -m) = arm64 ]]; then
-  # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr
-  platform+=(--platform linux/amd64)
-fi
-
-docker build "${platform[@]}" -t solanalabs/rust .
-
-read -r rustc version _ < <(docker run solanalabs/rust rustc --version)
-[[ $rustc = rustc ]]
-docker tag solanalabs/rust:latest solanalabs/rust:"$version"
-docker push solanalabs/rust:"$version"
-docker push solanalabs/rust:latest

ci/docker/Dockerfile

@@ -1,10 +1,12 @@
 FROM ubuntu:20.04
 
 ARG \
-  RUST_VERSION=1.75.0 \
+  RUST_VERSION= \
+  RUST_NIGHTLY_VERSION= \
   GOLANG_VERSION=1.21.3 \
   NODE_MAJOR=18 \
-  SCCACHE_VERSION=v0.5.4
+  SCCACHE_VERSION=v0.5.4 \
+  GRCOV_VERSION=v0.8.18
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
@@ -21,7 +23,10 @@ ENV \
   CARGO_HOME=/usr/local/cargo \
   PATH="$PATH:/usr/local/cargo/bin"
 
-RUN apt-get update && \
+RUN \
+  if [ -z "$RUST_VERSION" ]; then echo "ERROR: The RUST_VERSION argument is required!" && exit 1; fi && \
+  if [ -z "$RUST_NIGHTLY_VERSION" ]; then echo "ERROR: The RUST_NIGHTLY_VERSION argument is required!" && exit 1; fi && \
+  apt-get update && \
   apt-get install --no-install-recommends -y \
   # basic
   tzdata \
@@ -65,6 +70,9 @@ RUN apt-get update && \
   curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --no-modify-path --profile minimal --default-toolchain $RUST_VERSION -y && \
   rustup component add rustfmt && \
   rustup component add clippy && \
+  rustup install $RUST_NIGHTLY_VERSION && \
+  rustup component add clippy --toolchain=$RUST_NIGHTLY_VERSION && \
+  rustup component add rustfmt --toolchain=$RUST_NIGHTLY_VERSION && \
   rustup target add wasm32-unknown-unknown && \
   cargo install cargo-audit && \
   cargo install cargo-hack && \
@@ -74,6 +82,9 @@ RUN apt-get update && \
   cargo install svgbob_cli && \
   cargo install wasm-pack && \
   cargo install rustfilt && \
+  rustup show && \
+  rustc --version && \
+  cargo --version && \
   chmod -R a+w $CARGO_HOME $RUSTUP_HOME && \
   rm -rf $CARGO_HOME/registry && \
   # sccache
@@ -101,5 +112,14 @@ RUN apt-get update && \
   chmod -R a+w /.config && \
   mkdir /.npm && \
   chmod -R a+w /.npm && \
+  # grcov
+  curl -LOsS "https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/grcov-x86_64-unknown-linux-musl.tar.bz2" && \
+  tar -xf grcov-x86_64-unknown-linux-musl.tar.bz2 && \
+  mv ./grcov $CARGO_HOME/bin && \
+  rm grcov-x86_64-unknown-linux-musl.tar.bz2 && \
+  # codecov
+  curl -Os https://uploader.codecov.io/latest/linux/codecov && \
+  chmod +x codecov && \
+  mv codecov /usr/bin && \
   # clean lists
   rm -rf /var/lib/apt/lists/*

ci/docker/README.md

@@ -0,0 +1,11 @@
+Docker image containing rust, rust nightly and some preinstalled packages used in CI
+
+This image is manually maintained:
+
+#### CLI
+
+1. Edit
+   1. `ci/rust-version.sh` for rust and rust nightly version
+   2. `ci/docker/Dockerfile` for other packages
+2. Ensure you're a member of the [Solana Docker Hub Organization](https://hub.docker.com/u/solanalabs/) and already `docker login`
+3. Run `ci/docker/build.sh` to build/publish the new image

ci/docker/build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1091
+source "$here/../rust-version.sh"
+
+platform=()
+if [[ $(uname -m) = arm64 ]]; then
+  # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr
+  platform+=(--platform linux/amd64)
+fi
+
+echo "build image: ${ci_docker_image:?}"
+docker build "${platform[@]}" \
+  -f "$here/Dockerfile" \
+  --build-arg "RUST_VERSION=${rust_stable:?}" \
+  --build-arg "RUST_NIGHTLY_VERSION=${rust_nightly:?}" \
+  -t "$ci_docker_image" .
+
+docker push "$ci_docker_image"

ci/publish-crate.sh

@@ -72,7 +72,7 @@ for Cargo_toml in $Cargo_tomls; do
       echo "Attempt ${i} of ${numRetries}"
       # The rocksdb package does not build with the stock rust docker image so use
       # the solana rust docker image
-      if ci/docker-run.sh "$rust_stable_docker_image" bash -exc "cd $crate; $cargoCommand"; then
+      if ci/docker-run-default-image.sh bash -exc "cd $crate; $cargoCommand"; then
         break
       fi
 

ci/rust-version.sh

@@ -34,10 +34,10 @@ fi
 
 
 export rust_stable="$stable_version"
-export rust_stable_docker_image=solanalabs/rust:"$stable_version"
 
 export rust_nightly=nightly-"$nightly_version"
-export rust_nightly_docker_image=solanalabs/rust-nightly:"$nightly_version"
+
+export ci_docker_image="solanalabs/ci:rust_${rust_stable}_${rust_nightly}"
 
 [[ -z $1 ]] || (
 

ci/test.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Get the directory of the current script
+script_dir_by_bash_source=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+script_dir_by_0=$(cd "$(dirname "$0")" && pwd)
+
+echo "script_dir_by_bash_source = $script_dir_by_bash_source"
+echo "script_dir_by_0 = $script_dir_by_0"

docs/build.sh

@@ -6,12 +6,10 @@ cd "$(dirname "$0")"
 # shellcheck source=ci/env.sh
 source ../ci/env.sh
 
-: "${rust_stable_docker_image:=}" # Pacify shellcheck
-
 # shellcheck source=ci/rust-version.sh
 source ../ci/rust-version.sh
-../ci/docker-run.sh "$rust_stable_docker_image" docs/build-cli-usage.sh
-../ci/docker-run.sh "$rust_stable_docker_image" docs/convert-ascii-to-svg.sh
+../ci/docker-run-default-image.sh docs/build-cli-usage.sh
+../ci/docker-run-default-image.sh docs/convert-ascii-to-svg.sh
 ./set-solana-release-tag.sh
 
 # Get current channel

net/net.sh

@@ -191,7 +191,7 @@ build() {
   if [[ $(uname) != Linux || ! " ${supported[*]} " =~ $(lsb_release -sr) ]]; then
     # shellcheck source=ci/rust-version.sh
     source "$SOLANA_ROOT"/ci/rust-version.sh
-    MAYBE_DOCKER="ci/docker-run.sh $rust_stable_docker_image"
+    MAYBE_DOCKER="ci/docker-run.sh ${ci_docker_image:?}"
   fi
   SECONDS=0
   (

sdk/docker-solana/build.sh

@@ -20,8 +20,7 @@ fi
 
 cd "$(dirname "$0")"
 rm -rf usr/
-../../ci/docker-run.sh "$rust_stable_docker_image" \
-  scripts/cargo-install-all.sh sdk/docker-solana/usr
+../../ci/docker-run-default-image.sh scripts/cargo-install-all.sh sdk/docker-solana/usr
 
 cp -f ../../scripts/run.sh usr/bin/solana-run.sh
 cp -f ../../fetch-spl.sh usr/bin/