Overhaul cargo/rustc version management

ci/buildkite.yml

@@ -2,7 +2,7 @@ steps:
   - command: "ci/shellcheck.sh"
     name: "shellcheck"
     timeout_in_minutes: 5
-  - command: "ci/docker-run.sh solanalabs/rust:1.32.0 ci/test-checks.sh"
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-checks.sh"
     name: "checks"
     timeout_in_minutes: 15
   - wait
@@ -14,10 +14,10 @@ steps:
   - command: "ci/test-bench.sh"
     name: "bench"
     timeout_in_minutes: 20
-  - command: "ci/docker-run.sh solanalabs/rust:1.32.0 ci/test-stable.sh"
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
     name: "stable"
     timeout_in_minutes: 20
-  - command: "ci/docker-run.sh solanalabs/rust-nightly:2019-01-31 ci/test-coverage.sh"
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
     name: "coverage"
     timeout_in_minutes: 20
   # TODO: Fix and re-enable test-large-network.sh

ci/docker-rust-nightly/README.md

@@ -19,7 +19,7 @@ To update the pinned version:
    to confirm the new nightly image builds.  Fix any issues as needed
 1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized.
 1. Run `CI=true ci/docker-rust-nightly/build.sh YYYY-MM-DD` to push the new nightly image to dockerhub.com.
-1. Modify the `solanalabs/rust-nightly:YYYY-MM-DD` reference in `ci/buildkite.yml` from the previous to
+1. Modify the `solanalabs/rust-nightly:YYYY-MM-DD` reference in `ci/rust-version.sh` from the previous to
    new *YYYY-MM-DD* value, send a PR with this change and any codebase adjustments needed.
 
 ## Troubleshooting

ci/iterations-localnet.sh

@@ -24,9 +24,10 @@ fi
 
 build() {
   $genPipeline && return
-  ci/version-check-with-upgrade.sh stable
+  source ci/rust-version.sh stable
+
   _ scripts/ulimit-n.sh
-  _ cargo build --all
+  _ cargo +$rust_stable build --all
 }
 
 runTest() {

ci/publish-crate.sh

@@ -55,7 +55,8 @@ for crate in "${CRATES[@]}"; do
     set -x
     # TODO: the rocksdb package does not build with the stock rust docker image,
     # so use the solana rust docker image until this is resolved upstream
-    ci/docker-run.sh solanalabs/rust:1.32.0 bash -exc "cd $crate; $cargoCommand"
+    source ci/rust-version.sh
+    ci/docker-run.sh "$rust_stable_docker_image" bash -exc "cd $crate; $cargoCommand"
     #ci/docker-run.sh rust bash -exc "cd $crate; $cargoCommand"
   )
 done

ci/publish-tarball.sh

@@ -50,7 +50,8 @@ echo --- Creating tarball
     echo "target: $TARGET"
   ) > solana-release/version.yml
 
-  scripts/cargo-install-all.sh solana-release
+  source ci/rust-version.sh stable
+  scripts/cargo-install-all.sh +"$rust_stable" solana-release
 
   ./fetch-perf-libs.sh
   # shellcheck source=/dev/null

ci/rust-version.sh

@@ -0,0 +1,45 @@
+#
+# This file maintains the rust versions for use by CI.
+#
+# Build with stable rust, updating the stable toolchain if necessary:
+#   $ source ci/rust-version.sh stable
+#   $ cargo +"$rust_stable" build
+#
+# Build with nightly rust, updating the nightly toolchain if necessary:
+#   $ source ci/rust-version.sh nightly
+#   $ cargo +"$rust_nightly" build
+#
+# Obtain the environment variables without any automatic toolchain updating:
+#   $ source ci/rust-version.sh
+#
+
+export rust_stable=1.32.0
+export rust_stable_docker_image=solanalabs/rust:1.32.0
+
+export rust_nightly=nightly-2019-01-31
+export rust_nightly_docker_image=solanalabs/rust-nightly:2019-01-31
+
+[[ -z $1 ]] || (
+
+  rustup_install() {
+    declare toolchain=$1
+    if ! cargo +"$toolchain" -V; then
+      rustup install "$toolchain"
+      cargo +"$toolchain" -V
+    fi
+  }
+
+  set -e
+  cd "$(dirname "${BASH_SOURCE[0]}")"
+  case $1 in
+  stable)
+     rustup_install "$rust_stable"
+     ;;
+  nightly)
+     rustup_install "$rust_nightly"
+    ;;
+  *)
+    echo "Note: ignoring unknown argument: $1"
+    ;;
+  esac
+)

ci/test-bench.sh

@@ -24,7 +24,7 @@ source ci/_
 source ci/upload-ci-artifact.sh
 
 eval "$(ci/channel-info.sh)"
-ci/version-check-with-upgrade.sh nightly
+source ci/rust-version.sh nightly
 
 set -o pipefail
 export RUST_BACKTRACE=1
@@ -39,7 +39,7 @@ fi
 
 BENCH_FILE=bench_output.log
 BENCH_ARTIFACT=current_bench_results.log
-_ cargo +nightly bench ${V:+--verbose} \
+_ cargo +$rust_nightly bench ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee "$BENCH_FILE"
 
 # Run bpf benches
@@ -47,11 +47,11 @@ echo --- program/bpf
 (
   set -x
   cd programs/bpf
-  cargo +nightly bench ${V:+--verbose} --features=bpf_c \
+  cargo +$rust_nightly bench ${V:+--verbose} --features=bpf_c \
     -- -Z unstable-options --format=json --nocapture | tee -a ../../../"$BENCH_FILE"
 )
 
-_ cargo +nightly run --release --package solana-upload-perf \
+_ cargo +$rust_nightly run --release --package solana-upload-perf \
   -- "$BENCH_FILE" "$TARGET_BRANCH" "$UPLOAD_METRICS" > "$BENCH_ARTIFACT"
 
 upload-ci-artifact "$BENCH_ARTIFACT"

ci/test-checks.sh

@@ -4,14 +4,14 @@ set -e
 cd "$(dirname "$0")/.."
 
 source ci/_
-ci/version-check.sh stable
+source ci/rust-version.sh stable
 
 export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 
-_ cargo fmt --all -- --check
-_ cargo clippy --all -- --version
-_ cargo clippy --all -- --deny=warnings
+_ cargo +"$rust_stable" fmt --all -- --check
+_ cargo +"$rust_stable" clippy --all -- --version
+_ cargo +"$rust_stable" clippy --all -- --deny=warnings
 _ ci/audit.sh
 _ ci/nits.sh
 _ book/build.sh

ci/test-coverage.sh

@@ -21,7 +21,6 @@ ci/affects-files.sh \
 }
 
 source ci/upload-ci-artifact.sh
-ci/version-check-with-upgrade.sh nightly
 source scripts/ulimit-n.sh
 
 scripts/coverage.sh

ci/test-large-network.sh

@@ -4,9 +4,7 @@ set -e
 here=$(dirname "$0")
 cd "$here"/..
 
-# This job doesn't run within a container, try once to upgrade tooling on a
-# version check failure
-ci/version-check-with-upgrade.sh stable
+source ci/rust-version.sh stable
 
 export RUST_BACKTRACE=1
 
@@ -39,4 +37,4 @@ fi
 
 set -x
 export SOLANA_DYNAMIC_NODES=120
-exec cargo test --release --features=erasure test_multi_node_dynamic_network -- --ignored
+exec cargo +"$rust_stable" test --release --features=erasure test_multi_node_dynamic_network -- --ignored

ci/test-stable.sh

@@ -10,7 +10,8 @@ annotate() {
   }
 }
 
-ci/version-check-with-upgrade.sh stable
+source ci/rust-version.sh stable
+
 export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 source scripts/ulimit-n.sh
@@ -24,9 +25,9 @@ case $testName in
 test-stable)
   echo "Executing $testName"
 
-  _ cargo build --all ${V:+--verbose}
-  _ cargo test --all ${V:+--verbose} -- --nocapture --test-threads=1
-  _ cargo test --manifest-path runtime/Cargo.toml
+  _ cargo +"$rust_stable" build --all ${V:+--verbose}
+  _ cargo +"$rust_stable" test --all ${V:+--verbose} -- --nocapture --test-threads=1
+  _ cargo +"$rust_stable" test --manifest-path runtime/Cargo.toml
   ;;
 test-stable-perf)
   echo "Executing $testName"
@@ -48,7 +49,9 @@ test-stable-perf)
   # BPF program tests
   _ make -C programs/bpf/c tests
   _ programs/bpf/rust/noop/build.sh # Must be built out of band
-  _ cargo test --manifest-path programs/bpf/Cargo.toml --no-default-features --features=bpf_c,bpf_rust
+  _ cargo +"$rust_stable" test \
+    --manifest-path programs/bpf/Cargo.toml \
+    --no-default-features --features=bpf_c,bpf_rust
 
   # Run root package tests with these features
   ROOT_FEATURES=erasure,chacha
@@ -67,9 +70,9 @@ test-stable-perf)
   fi
 
   # Run root package library tests
-  _ cargo build --all ${V:+--verbose} --features="$ROOT_FEATURES"
-  _ cargo test --all --lib ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1
-  _ cargo test --manifest-path runtime/Cargo.toml
+  _ cargo +"$rust_stable" build --all ${V:+--verbose} --features="$ROOT_FEATURES"
+  _ cargo +"$rust_stable" test --all --lib ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1
+  _ cargo +"$rust_stable" test --manifest-path runtime/Cargo.toml
 
   # Run root package integration tests
   for test in tests/*.rs; do
@@ -77,7 +80,7 @@ test-stable-perf)
     test=${test%.rs} # basename x .rs
     (
       export RUST_LOG="$test"=trace,$RUST_LOG
-      _ cargo test --all ${V:+--verbose} --features="$ROOT_FEATURES" --test="$test" \
+      _ cargo +"$rust_stable" test --all ${V:+--verbose} --features="$ROOT_FEATURES" --test="$test" \
         -- --test-threads=1 --nocapture
     )
   done

ci/version-check-with-upgrade.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-cd "$(dirname "$0")"
-
-channel=${1:-stable}
-if ! ./version-check.sh "$channel"; then
-  rustup install "$channel"
-  ./version-check.sh "$channel"
-fi

ci/version-check.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-require() {
-  declare expectedProgram="$1"
-  declare expectedVersion="$2"
-  shift 2
-
-  read -r program version _ < <($expectedProgram "$@" -V)
-
-  declare ok=true
-  [[ $program = "$expectedProgram" ]] || ok=false
-  [[ $version =~ $expectedVersion ]] || ok=false
-
-  echo "Found $program $version"
-  if ! $ok; then
-    echo Error: expected "$expectedProgram $expectedVersion"
-    exit 1
-  fi
-}
-
-case ${1:-stable} in
-nightly)
-  require rustc 1.34.[0-9]+-nightly +nightly
-  require cargo 1.34.[0-9]+-nightly +nightly
-  ;;
-stable)
-  require rustc 1.32.[0-9]+
-  require cargo 1.32.[0-9]+
-  ;;
-*)
-  echo Error: unknown argument: "$1"
-  exit 1
-  ;;
-esac
-
-exit 0

net/net.sh

@@ -117,7 +117,8 @@ loadConfigFile
 build() {
   declare MAYBE_DOCKER=
   if [[ $(uname) != Linux ]]; then
-    MAYBE_DOCKER="ci/docker-run.sh solanalabs/rust"
+    source ci/rust-version.sh
+    MAYBE_DOCKER="ci/docker-run.sh +$rust_stable_docker_image"
   fi
   SECONDS=0
   (

scripts/cargo-install-all.sh

@@ -4,10 +4,17 @@
 # other workspace crates or native program crates.
 set -e
 
+export rust_version=
+if [[ $1 =~ \+ ]]; then
+  export rust_version=$1
+  shift
+fi
+
 if [[ -z $1 ]]; then
   echo Install directory not specified
   exit 1
 fi
+
 installDir="$(mkdir -p "$1"; cd "$1"; pwd)"
 cargoFeatures="$2"
 echo "Install location: $installDir"
@@ -35,7 +42,7 @@ BIN_CRATES=(
 for crate in "${BIN_CRATES[@]}"; do
   (
     set -x
-    cargo install --force --path "$crate" --root "$installDir" --features="$cargoFeatures"
+    cargo "$rust_version" install --force --path "$crate" --root "$installDir" --features="$cargoFeatures"
   )
 done
 

scripts/coverage.sh

@@ -28,8 +28,9 @@ if [[ -d target/cov ]]; then
 fi
 rm -rf target/cov/$reportName
 
-_ cargo +nightly build --target-dir target/cov --all
-_ cargo +nightly test --target-dir target/cov --lib --all -- --test-threads=1
+source ci/rust-version.sh nightly
+_ cargo +$rust_nightly build --target-dir target/cov --all
+_ cargo +$rust_nightly test --target-dir target/cov --lib --all -- --test-threads=1
 
 _ scripts/fetch-grcov.sh
 echo "--- grcov"

sdk/docker-solana/build.sh

@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
 set -ex
 
-cd "$(dirname "$0")"
-eval "$(../../ci/channel-info.sh)"
+cd "$(dirname "$0")"/../..
+eval "$(ci/channel-info.sh)"
+source ci/rust-version.sh
 
 if [[ -z $CHANNEL ]]; then
   echo Unable to determine channel to publish into, exiting.
@@ -10,8 +11,9 @@ if [[ -z $CHANNEL ]]; then
   exit 0
 fi
 
+cd "$(dirname "$0")"
 rm -rf usr/
-../../ci/docker-run.sh solanalabs/rust:1.32.0 \
+../../ci/docker-run.sh "$rust_stable_docker_image" \
   scripts/cargo-install-all.sh sdk/docker-solana/usr
 
 cp -f ../../run.sh usr/bin/solana-run.sh