Introduce normalized CI environment vars: ci/env.sh (#4571)

.buildkite/hooks/post-checkout

@@ -1,6 +1,8 @@
 CI_BUILD_START=$(date +%s)
 export CI_BUILD_START
 
+source ci/env.sh
+
 #
 # Kill any running docker containers, which are potentially left over from the
 # previous CI job

ci/affects-files.sh

@@ -12,7 +12,7 @@
 set -e
 cd "$(dirname "$0")"/..
 
-if ci/is-pr.sh; then
+if [[ -n $CI_PULL_REQUEST ]]; then
   affectedFiles="$(buildkite-agent meta-data get affected_files)"
   echo "Affected files in this PR: $affectedFiles"
 

ci/channel-info.sh

@@ -89,11 +89,11 @@ BETA_CHANNEL_LATEST_TAG=${beta_tag:+v$beta_tag}
 STABLE_CHANNEL_LATEST_TAG=${stable_tag:+v$stable_tag}
 
 
-if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
+if [[ $CI_BRANCH = "$STABLE_CHANNEL" ]]; then
   CHANNEL=stable
-elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
+elif [[ $CI_BRANCH = "$EDGE_CHANNEL" ]]; then
   CHANNEL=edge
-elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
+elif [[ $CI_BRANCH = "$BETA_CHANNEL" ]]; then
   CHANNEL=beta
 fi
 

ci/docker-run.sh

@@ -64,11 +64,14 @@ fi
 ARGS+=(
   --env BUILDKITE
   --env BUILDKITE_AGENT_ACCESS_TOKEN
-  --env BUILDKITE_BRANCH
-  --env BUILDKITE_COMMIT
   --env BUILDKITE_JOB_ID
-  --env BUILDKITE_TAG
   --env CI
+  --env CI_BRANCH
+  --env CI_BUILD_ID
+  --env CI_COMMIT
+  --env CI_JOB_ID
+  --env CI_PULL_REQUEST
+  --env CI_REPO_SLUG
   --env CODECOV_TOKEN
   --env CRATES_IO_TOKEN
 )

ci/env.sh

@@ -0,0 +1,59 @@
+#
+# Normalized CI environment variables
+#
+# |source| me
+#
+
+if ${CI:-false}; then
+  if ${TRAVIS:-false}; then
+    export CI_BRANCH=$TRAVIS_BRANCH
+    export CI_BUILD_ID=$TRAVIS_BUILD_ID
+    export CI_COMMIT=$TRAVIS_COMMIT
+    export CI_JOB_ID=$TRAVIS_JOB_ID
+    if $TRAVIS_PULL_REQUEST; then
+      export CI_PULL_REQUEST=true
+    else
+      export CI_PULL_REQUEST=
+    fi
+    export CI_TAG=$TRAVIS_TAG
+  fi
+  if ${BUILDKITE:-false}; then
+    export CI_BRANCH=$BUILDKITE_BRANCH
+    export CI_BUILD_ID=$BUILDKITE_BUILD_ID
+    export CI_COMMIT=$BUILDKITE_COMMIT
+    export CI_JOB_ID=$BUILDKITE_JOB_ID
+    # The standard BUILDKITE_PULL_REQUEST environment variable is always "false" due
+    # to how solana-ci-gate is used to trigger PR builds rather than using the
+    # standard Buildkite PR trigger.
+    if [[ $CI_BRANCH =~ pull/* ]]; then
+      export CI_PULL_REQUEST=true
+    else
+      export CI_PULL_REQUEST=
+    fi
+    # TRIGGERED_BUILDKITE_TAG is a workaround to propagate BUILDKITE_TAG into
+    # the solana-secondary builder
+    if [[ -n $TRIGGERED_BUILDKITE_TAG ]]; then
+      export CI_TAG=$TRIGGERED_BUILDKITE_TAG
+    else
+      export CI_TAG=$BUILDKITE_TAG
+    fi
+  fi
+else
+  export CI=
+  export CI_BRANCH=
+  export CI_BUILD_ID=
+  export CI_COMMIT=
+  export CI_JOB_ID=
+  export CI_PULL_REQUEST=
+  export CI_TAG=
+fi
+
+cat <<EOF
+CI=$CI
+CI_BRANCH=$CI_BRANCH
+CI_BUILD_ID=$CI_BUILD_ID
+CI_COMMIT=$CI_COMMIT
+CI_JOB_ID=$CI_JOB_ID
+CI_PULL_REQUEST=$CI_PULL_REQUEST
+CI_TAG=$CI_TAG
+EOF

ci/is-pr.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-set -e
-#
-# The standard BUILDKITE_PULL_REQUEST environment variable is always "false" due
-# to how solana-ci-gate is used to trigger PR builds rather than using the
-# standard Buildkite PR trigger.
-#
-
-[[ $BUILDKITE_BRANCH =~ pull/* ]]

ci/publish-book.sh

@@ -13,7 +13,7 @@ echo --- create book repo
   git config user.email "maintainers@solana.com"
   git config user.name "$(basename "$0")"
   git add ./* ./.nojekyll
-  git commit -m "${BUILDKITE_COMMIT:-local}"
+  git commit -m "${CI_COMMIT:-local}"
 )
 
 eval "$(ci/channel-info.sh)"

ci/publish-crate.sh

@@ -12,12 +12,12 @@ is_crate_version_uploaded() {
 }
 
 # Only package/publish if this is a tagged release
-[[ -n $TRIGGERED_BUILDKITE_TAG ]] || {
+[[ -n $CI_TAG ]] || {
-  echo TRIGGERED_BUILDKITE_TAG unset, skipped
+  echo CI_TAG unset, skipped
   exit 0
 }
 
-semverParseInto "$TRIGGERED_BUILDKITE_TAG" MAJOR MINOR PATCH SPECIAL
+semverParseInto "$CI_TAG" MAJOR MINOR PATCH SPECIAL
 expectedCrateVersion="$MAJOR.$MINOR.$PATCH$SPECIAL"
 
 [[ -n "$CRATES_IO_TOKEN" ]] || {

ci/publish-metrics-dashboard.sh

@@ -55,7 +55,7 @@ stable)
   ;;
 esac
 
-if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
+if [[ $CI_BRANCH != "$CHANNEL_BRANCH" ]]; then
   (
     cat <<EOF
 steps:

ci/publish-tarball.sh

@@ -4,7 +4,7 @@ set -e
 cd "$(dirname "$0")/.."
 
 DRYRUN=
-if [[ -z $BUILDKITE_BRANCH ]]; then
+if [[ -z $CI_BRANCH ]]; then
   DRYRUN="echo"
   CHANNEL=unknown
 fi
@@ -12,12 +12,9 @@ fi
 eval "$(ci/channel-info.sh)"
 
 TAG=
-if [[ -n "$BUILDKITE_TAG" ]]; then
+if [[ -n "$CI_TAG" ]]; then
-  CHANNEL_OR_TAG=$BUILDKITE_TAG
+  CHANNEL_OR_TAG=$CI_TAG
-  TAG="$BUILDKITE_TAG"
+  TAG="$CI_TAG"
-elif [[ -n "$TRIGGERED_BUILDKITE_TAG" ]]; then
-  CHANNEL_OR_TAG=$TRIGGERED_BUILDKITE_TAG
-  TAG="$TRIGGERED_BUILDKITE_TAG"
 else
   CHANNEL_OR_TAG=$CHANNEL
 fi

ci/test-bench.sh

@@ -30,8 +30,8 @@ set -o pipefail
 export RUST_BACKTRACE=1
 
 UPLOAD_METRICS=""
-TARGET_BRANCH=$BUILDKITE_BRANCH
+TARGET_BRANCH=$CI_BRANCH
-if [[ -z $BUILDKITE_BRANCH ]] || ./ci/is-pr.sh; then
+if [[ -z $CI_BRANCH ]] || [[ -n $CI_PULL_REQUEST ]]; then
   TARGET_BRANCH=$EDGE_CHANNEL
 else
   UPLOAD_METRICS="upload"

ci/test-coverage.sh

@@ -25,7 +25,7 @@ source scripts/ulimit-n.sh
 
 scripts/coverage.sh
 
-report=coverage-"${BUILDKITE_COMMIT:0:9}".tar.gz
+report=coverage-"${CI_COMMIT:0:9}".tar.gz
 mv target/cov/report.tar.gz "$report"
 upload-ci-artifact "$report"
 annotate --style success --context lcov-report \
@@ -39,5 +39,5 @@ else
   bash <(curl -s https://codecov.io/bash) -X gcov -f target/cov/lcov.info
 
   annotate --style success --context codecov.io \
-    "CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${BUILDKITE_COMMIT:0:9}"
+    "CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${CI_COMMIT:0:9}"
 fi

ci/testnet-manager.sh

@@ -184,7 +184,7 @@ if [[ -n $TESTNET_TAG ]]; then
   CHANNEL_OR_TAG=$TESTNET_TAG
 else
 
-  if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
+  if [[ $CI_BRANCH != "$CHANNEL_BRANCH" ]]; then
     (
       cat <<EOF
 steps:

ci/upload-github-release-asset.sh

@@ -20,26 +20,20 @@ if [[ -z $GITHUB_TOKEN ]]; then
   exit 1
 fi
 
-if [[ -n $BUILDKITE_TAG ]]; then
+if [[ -z $CI_TAG ]]; then
-  TAG=$BUILDKITE_TAG
+  echo Error: CI_TAG not defined
-elif [[ -n $TRIGGERED_BUILDKITE_TAG ]]; then
-  TAG=$TRIGGERED_BUILDKITE_TAG
-fi
-
-if [[ -z $TAG ]]; then
-  echo Error: TAG not defined
   exit 1
 fi
 
 releaseId=$( \
-  curl -s "https://api.github.com/repos/$REPO_SLUG/releases/tags/$TAG" \
+  curl -s "https://api.github.com/repos/$REPO_SLUG/releases/tags/$CI_TAG" \
   | grep -m 1 \"id\": \
   | sed -ne 's/^[^0-9]*\([0-9]*\),$/\1/p' \
 )
-echo "Github release id for $TAG is $releaseId"
+echo "Github release id for $CI_TAG is $releaseId"
 
 for file in "$@"; do
-  echo "--- Uploading $file to tag $TAG of $REPO_SLUG"
+  echo "--- Uploading $file to tag $CI_TAG of $REPO_SLUG"
   curl \
     --data-binary @"$file" \
     -H "Authorization: token $GITHUB_TOKEN" \

scripts/coverage.sh

@@ -9,8 +9,8 @@ set -e
 cd "$(dirname "$0")/.."
 source ci/_
 
-: "${BUILDKITE_COMMIT:=local}"
+: "${CI_COMMIT:=local}"
-reportName="lcov-${BUILDKITE_COMMIT:0:9}"
+reportName="lcov-${CI_COMMIT:0:9}"
 
 if [[ -n $1 ]]; then
   crate=--manifest-path=$1/Cargo.toml