Rework buildkite pipeline construction to be more composable

.buildkite/pipeline-upload.sh

@@ -9,23 +9,10 @@
 
 set -e
 cd "$(dirname "$0")"/..
+source ci/_
 
-if [[ -n $BUILDKITE_TAG ]]; then
+_ ci/buildkite/pipeline.sh pipeline.yml
-  buildkite-agent annotate --style info --context release-tag \
+echo +++ pipeline
-    "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG"
+cat pipeline.yml
-  buildkite-agent pipeline upload ci/buildkite-release.yml
-else
-  if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
-    # Add helpful link back to the corresponding Github Pull Request
-    buildkite-agent annotate --style info --context pr-backlink \
-      "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
-  fi
 
-  if [[ $BUILDKITE_MESSAGE =~ GitBook: ]]; then
+_ buildkite-agent pipeline upload pipeline.yml
-    buildkite-agent annotate --style info --context gitbook-ci-skip \
-      "GitBook commit detected, CI skipped"
-    exit
-  fi
-
-  buildkite-agent pipeline upload ci/buildkite.yml
-fi

ci/buildkite-release.yml

@@ -1,15 +0,0 @@
-# Build steps that run on a release tag
-#
-# All the steps in `buildkite.yml` are skipped and we jump directly to the
-# secondary build steps since it's assumed the commit that was tagged is known
-# to be good so there's no need to rebuild and retest it.
-steps:
-  - trigger: "solana-secondary"
-    branches: "!pull/*"
-    async: true
-    build:
-      message: "${BUILDKITE_MESSAGE}"
-      commit: "${BUILDKITE_COMMIT}"
-      branch: "${BUILDKITE_BRANCH}"
-      env:
-        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"

ci/buildkite.yml

@@ -1,38 +0,0 @@
-# Build steps that run on pushes and pull requests.
-# If files other than those in docs/ were modified, this will be followed up by
-# ci/buildkite-tests.yml
-#
-# Release tags use buildkite-release.yml instead
-
-steps:
-  - command: "ci/dependabot-pr.sh"
-    name: "dependabot"
-    timeout_in_minutes: 5
-    if: build.env("GITHUB_USER") == "dependabot-preview[bot]"
-
-  - wait
-
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh"
-    name: "checks"
-    timeout_in_minutes: 20
-  - command: "ci/shellcheck.sh"
-    name: "shellcheck"
-    timeout_in_minutes: 5
-
-  - wait
-
-  - command: "ci/maybe-trigger-tests.sh"
-    name: "maybe-trigger-tests"
-    timeout_in_minutes: 2
-
-  - wait
-
-  - trigger: "solana-secondary"
-    branches: "!pull/*"
-    async: true
-    build:
-      message: "${BUILDKITE_MESSAGE}"
-      commit: "${BUILDKITE_COMMIT}"
-      branch: "${BUILDKITE_BRANCH}"
-      env:
-        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"

ci/buildkite/all-tests.yml

@@ -1,7 +1,7 @@
-# These steps are conditionally triggered by ci/buildkite.yml when files
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh"
-# other than those in docs/ are modified
+    name: "checks"
-
+    timeout_in_minutes: 20
-steps:
+  - wait
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
     name: "coverage"
     timeout_in_minutes: 30

ci/buildkite/dependabot-pr.yml

@@ -0,0 +1,6 @@
+  - command: "ci/dependabot-pr.sh"
+    name: "dependabot"
+    timeout_in_minutes: 5
+    if: build.env("GITHUB_USER") == "dependabot-preview[bot]"
+
+  - wait

ci/buildkite/docs.yml

@@ -0,0 +1,2 @@
+  - command: "docs/build.sh"
+    timeout_in_minutes: 5

ci/buildkite/pipeline.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+#
+# Builds a buildkite pipeline based on the environment variables
+#
+
+set -e
+cd "$(dirname "$0")"/../..
+
+output_file=${1:-/dev/stderr}
+
+start_pipeline() {
+  echo "# $*" > "$output_file"
+  echo "steps:" >> "$output_file"
+}
+
+wait_step() {
+  echo "  - wait" >> "$output_file"
+}
+
+cat_steps() {
+  cat "$@" >> "$output_file"
+}
+
+pull_or_push_steps() {
+  cat_steps ci/buildkite/sanity.yml
+  wait_step
+
+  if ci/affects-files.sh .sh$; then
+    cat_steps ci/buildkite/shellcheck.yml
+  fi
+  wait_step
+
+  docs=false
+  all_tests=false
+
+  if ci/affects-files.sh ^docs/; then
+    docs=true
+  fi
+
+  if ci/affects-files.sh !^docs/ !.md$ !^.buildkite/; then
+    all_tests=true
+  fi
+
+  if $docs; then
+    cat_steps ci/buildkite/docs.yml
+  fi
+
+  if $all_tests; then
+    cat_steps ci/buildkite/all-tests.yml
+  fi
+}
+
+
+if [[ -n $BUILDKITE_TAG ]]; then
+  start_pipeline "Tag pipeline for $BUILDKITE_TAG"
+
+  buildkite-agent annotate --style info --context release-tag \
+    "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG"
+
+  # Jump directly to the secondary build to publish release artifacts quickly
+  cat_steps ci/buildkite/trigger-secondary.yml
+  exit 0
+fi
+
+
+if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
+  start_pipeline "Pull request pipeline for $BUILDKITE_BRANCH"
+
+  # Add helpful link back to the corresponding Github Pull Request
+  buildkite-agent annotate --style info --context pr-backlink \
+    "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
+
+  cat_steps ci/buildkite/dependabot-pr.yml
+  pull_or_push_steps
+  exit 0
+fi
+
+start_pipeline "Push pipeline for $BUILDKITE_BRANCH"
+pull_or_push_steps
+wait_step
+cat_steps ci/buildkite/trigger-secondary.yml
+exit 0

ci/buildkite/sanity.yml

@@ -0,0 +1,3 @@
+  - command: "ci/test-sanity.sh"
+    name: "sanity"
+    timeout_in_minutes: 5

ci/buildkite/shellcheck.yml

@@ -0,0 +1,3 @@
+  - command: "ci/shellcheck.sh"
+    name: "shellcheck"
+    timeout_in_minutes: 5

ci/buildkite/trigger-secondary.yml

@@ -0,0 +1,9 @@
+  - trigger: "solana-secondary"
+    branches: "!pull/*"
+    async: true
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"

ci/maybe-trigger-tests.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-set -e
-cd "$(dirname "$0")/.."
-
-annotate() {
-  ${BUILDKITE:-false} && {
-    buildkite-agent annotate "$@"
-  }
-}
-
-# Skip if only the docs have been modified
-ci/affects-files.sh \
-  \!^docs/ \
-|| {
-  annotate --style info \
-    "Skipping all further tests as only docs/ files were modified"
-  exit 0
-}
-
-annotate --style info "Triggering tests"
-buildkite-agent pipeline upload ci/buildkite-tests.yml

ci/test-checks.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+#
+# cargo fmt, cargo clippy
+#
 set -e
 
 cd "$(dirname "$0")/.."
@@ -10,9 +13,6 @@ source ci/rust-version.sh nightly
 export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 
-# Look for failed mergify.io backports
-_ git show HEAD --check --oneline
-
 if _ scripts/cargo-for-all-lock-files.sh +"$rust_nightly" check --locked --all-targets; then
   true
 else
@@ -22,6 +22,7 @@ else
   exit "$check_status"
 fi
 
+_ ci/order-crates-for-publishing.py
 _ cargo +"$rust_stable" fmt --all -- --check
 
 # -Z... is needed because of clippy bug: https://github.com/rust-lang/rust-clippy/issues/4612
@@ -30,10 +31,6 @@ _ cargo +"$rust_nightly" clippy -Zunstable-options --workspace --all-targets --
 
 _ cargo +"$rust_stable" audit --version
 _ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit --ignore RUSTSEC-2020-0002 --ignore RUSTSEC-2020-0008
-_ ci/nits.sh
-_ ci/order-crates-for-publishing.py
-_ docs/build.sh
-_ ci/check-ssh-keys.sh
 
 {
   cd programs/bpf

ci/test-sanity.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")/.."
+
+source ci/_
+
+# Look for failed mergify.io backports
+_ git show HEAD --check --oneline
+
+_ ci/nits.sh
+_ ci/check-ssh-keys.sh
+
+echo --- ok