Buildkite pipeline overhaul 2 (#10529)
This commit is contained in:
Michael Vines
GitHub
parent
fb8612be49
commit
206b8118a1
|
|
@ -11,7 +11,7 @@ set -e
|
|||
cd "$(dirname "$0")"/..
|
||||
source ci/_
|
||||
|
||||
_ ci/buildkite/pipeline.sh pipeline.yml
|
||||
_ ci/buildkite-pipeline.sh pipeline.yml
|
||||
echo +++ pipeline
|
||||
cat pipeline.yml
|
||||
|
||||
|
|
|
|||
|
|
@ -1,43 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Checks if a CI build affects one or more path patterns. Each command-line
|
||||
# argument is checked in series.
|
||||
#
|
||||
# Bash regular expressions are permitted in the pattern:
|
||||
# ./affects-files.sh .rs$ -- any file or directory ending in .rs
|
||||
# ./affects-files.sh .rs -- also matches foo.rs.bar
|
||||
# ./affects-files.sh ^snap/ -- anything under the snap/ subdirectory
|
||||
# ./affects-files.sh snap/ -- also matches foo/snap/
|
||||
# Any pattern starting with the ! character will be negated:
|
||||
# ./affects-files.sh !^docs/ -- anything *not* under the docs/ subdirectory
|
||||
#
|
||||
set -e
|
||||
cd "$(dirname "$0")"/..
|
||||
|
||||
if [[ -n $CI_PULL_REQUEST ]]; then
|
||||
affectedFiles="$(buildkite-agent meta-data get affected_files)"
|
||||
echo "Affected files in this PR: $affectedFiles"
|
||||
|
||||
IFS=':' read -ra files <<< "$affectedFiles"
|
||||
for pattern in "$@"; do
|
||||
if [[ ${pattern:0:1} = "!" ]]; then
|
||||
for file in "${files[@]}"; do
|
||||
if [[ ! $file =~ ${pattern:1} ]]; then
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
else
|
||||
for file in "${files[@]}"; do
|
||||
if [[ $file =~ $pattern ]]; then
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# affected_files metadata is not currently available for non-PR builds, so assume
|
||||
# the worse (affected)
|
||||
exit 0
|
||||
|
|
@ -0,0 +1,260 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Builds a buildkite pipeline based on the environment variables
|
||||
#
|
||||
|
||||
set -e
|
||||
cd "$(dirname "$0")"/..
|
||||
|
||||
output_file=${1:-/dev/stderr}
|
||||
|
||||
if [[ -n $CI_PULL_REQUEST ]]; then
|
||||
IFS=':' read -ra affected_files <<< "$(buildkite-agent meta-data get affected_files)"
|
||||
if [[ ${#affected_files[*]} -eq 0 ]]; then
|
||||
echo "Unable to determine the files affected by this PR"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
affected_files=()
|
||||
fi
|
||||
|
||||
annotate() {
|
||||
if [[ -n $BUILDKITE ]]; then
|
||||
buildkite-agent annotate "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# Checks if a CI pull request affects one or more path patterns. Each
|
||||
# argument is checked in series.
|
||||
#
|
||||
# Bash regular expressions are permitted in the pattern:
|
||||
# affects .rs$ -- any file or directory ending in .rs
|
||||
# affects .rs -- also matches foo.rs.bar
|
||||
# affects ^snap/ -- anything under the snap/ subdirectory
|
||||
# affects snap/ -- also matches foo/snap/
|
||||
# Any pattern starting with the ! character will be negated:
|
||||
# affects !^docs/ -- anything *not* under the docs/ subdirectory
|
||||
#
|
||||
affects() {
|
||||
if [[ -z $CI_PULL_REQUEST ]]; then
|
||||
# affected_files metadata is not currently available for non-PR builds so assume
|
||||
# the worse (affected)
|
||||
return 0
|
||||
fi
|
||||
for pattern in "$@"; do
|
||||
if [[ ${pattern:0:1} = "!" ]]; then
|
||||
for file in "${affected_files[@]}"; do
|
||||
if [[ ! $file =~ ${pattern:1} ]]; then
|
||||
return 0 # affected
|
||||
fi
|
||||
done
|
||||
else
|
||||
for file in "${affected_files[@]}"; do
|
||||
if [[ $file =~ $pattern ]]; then
|
||||
return 0 # affected
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
return 1 # not affected
|
||||
}
|
||||
|
||||
|
||||
# Checks if a CI pull request affects anything other than the provided path patterns
|
||||
#
|
||||
# Syntax is the same as `affects()` except that the negation prefix is not
|
||||
# supported
|
||||
#
|
||||
affects_other_than() {
|
||||
if [[ -z $CI_PULL_REQUEST ]]; then
|
||||
# affected_files metadata is not currently available for non-PR builds so assume
|
||||
# the worse (affected)
|
||||
return 0
|
||||
fi
|
||||
|
||||
for file in "${affected_files[@]}"; do
|
||||
declare matched=false
|
||||
for pattern in "$@"; do
|
||||
if [[ $file =~ $pattern ]]; then
|
||||
matched=true
|
||||
fi
|
||||
done
|
||||
if ! $matched; then
|
||||
return 0 # affected
|
||||
fi
|
||||
done
|
||||
|
||||
return 1 # not affected
|
||||
}
|
||||
|
||||
|
||||
start_pipeline() {
|
||||
echo "# $*" > "$output_file"
|
||||
echo "steps:" >> "$output_file"
|
||||
}
|
||||
|
||||
command_step() {
|
||||
cat >> "$output_file" <<EOF
|
||||
- name: "$1"
|
||||
command: "$2"
|
||||
timeout_in_minutes: $3
|
||||
artifact_paths: "log-*.txt"
|
||||
EOF
|
||||
}
|
||||
|
||||
|
||||
trigger_secondary_step() {
|
||||
cat >> "$output_file" <<"EOF"
|
||||
- trigger: "solana-secondary"
|
||||
branches: "!pull/*"
|
||||
async: true
|
||||
build:
|
||||
message: "${BUILDKITE_MESSAGE}"
|
||||
commit: "${BUILDKITE_COMMIT}"
|
||||
branch: "${BUILDKITE_BRANCH}"
|
||||
env:
|
||||
TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"
|
||||
EOF
|
||||
}
|
||||
|
||||
wait_step() {
|
||||
echo " - wait" >> "$output_file"
|
||||
}
|
||||
|
||||
all_test_steps() {
|
||||
command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20
|
||||
wait_step
|
||||
|
||||
# Coverage...
|
||||
if affects \
|
||||
.rs$ \
|
||||
Cargo.lock$ \
|
||||
Cargo.toml$ \
|
||||
^ci/rust-version.sh \
|
||||
^ci/test-coverage.sh \
|
||||
^scripts/coverage.sh \
|
||||
; then
|
||||
command_step coverge ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 30
|
||||
wait_step
|
||||
else
|
||||
annotate --style info --context test-coverage \
|
||||
"Coverage skipped as no .rs files were modified"
|
||||
fi
|
||||
|
||||
# Full test suite
|
||||
command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
|
||||
wait_step
|
||||
|
||||
# Perf test suite
|
||||
if affects \
|
||||
.rs$ \
|
||||
Cargo.lock$ \
|
||||
Cargo.toml$ \
|
||||
^ci/rust-version.sh \
|
||||
^ci/test-stable-perf.sh \
|
||||
^ci/test-stable.sh \
|
||||
^ci/test-local-cluster.sh \
|
||||
^core/build.rs \
|
||||
^fetch-perf-libs.sh \
|
||||
^programs/ \
|
||||
^sdk/ \
|
||||
; then
|
||||
cat >> "$output_file" <<"EOF"
|
||||
- command: "ci/test-stable-perf.sh"
|
||||
name: "stable-perf"
|
||||
timeout_in_minutes: 40
|
||||
artifact_paths: "log-*.txt"
|
||||
agents:
|
||||
- "queue=cuda"
|
||||
EOF
|
||||
else
|
||||
annotate --style info \
|
||||
"Stable-perf skipped as no relevant files were modified"
|
||||
fi
|
||||
|
||||
# Benches...
|
||||
if affects \
|
||||
.rs$ \
|
||||
Cargo.lock$ \
|
||||
Cargo.toml$ \
|
||||
^ci/rust-version.sh \
|
||||
^ci/test-coverage.sh \
|
||||
^ci/test-bench.sh \
|
||||
; then
|
||||
command_step bench "ci/test-bench.sh" 30
|
||||
else
|
||||
annotate --style info --context test-bench \
|
||||
"Bench skipped as no .rs files were modified"
|
||||
fi
|
||||
|
||||
command_step "local-cluster" \
|
||||
". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster.sh" \
|
||||
45
|
||||
}
|
||||
|
||||
pull_or_push_steps() {
|
||||
command_step sanity "ci/test-sanity.sh" 5
|
||||
wait_step
|
||||
|
||||
# Check for any .sh file changes
|
||||
if affects .sh$; then
|
||||
command_step shellcheck "ci/shellcheck.sh" 5
|
||||
wait_step
|
||||
fi
|
||||
|
||||
# Run the full test suite by default, skipping only if modifications are local
|
||||
# to some particular areas of the tree
|
||||
if affects_other_than ^.buildkite/ .md$ ^docs/ ^web3.js/; then
|
||||
all_test_steps
|
||||
fi
|
||||
|
||||
# doc/ changes:
|
||||
if affects ^docs/; then
|
||||
command_step docs ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image docs/build.sh" 5
|
||||
fi
|
||||
|
||||
# web3.js/ changes:
|
||||
if affects ^web3.js/; then
|
||||
echo "# TODO: run solana-web3.js tests..." >> "$output_file"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
if [[ -n $BUILDKITE_TAG ]]; then
|
||||
start_pipeline "Tag pipeline for $BUILDKITE_TAG"
|
||||
|
||||
annotate --style info --context release-tag \
|
||||
"https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG"
|
||||
|
||||
# Jump directly to the secondary build to publish release artifacts quickly
|
||||
trigger_secondary_step
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
|
||||
echo "+++ Affected files in this PR"
|
||||
for file in "${affected_files[@]}"; do
|
||||
echo "- $file"
|
||||
done
|
||||
|
||||
start_pipeline "Pull request pipeline for $BUILDKITE_BRANCH"
|
||||
|
||||
# Add helpful link back to the corresponding Github Pull Request
|
||||
annotate --style info --context pr-backlink \
|
||||
"Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
|
||||
|
||||
if [[ $GITHUB_USER = "dependabot-preview[bot]" ]]; then
|
||||
command_step dependabot "ci/dependabot-pr.sh" 5
|
||||
wait_step
|
||||
fi
|
||||
pull_or_push_steps
|
||||
exit 0
|
||||
fi
|
||||
|
||||
start_pipeline "Push pipeline for ${BUILDKITE_BRANCH:-?unknown branch?}"
|
||||
pull_or_push_steps
|
||||
wait_step
|
||||
trigger_secondary_step
|
||||
exit 0
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
- command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh"
|
||||
name: "checks"
|
||||
timeout_in_minutes: 20
|
||||
- wait
|
||||
- command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
|
||||
name: "coverage"
|
||||
timeout_in_minutes: 30
|
||||
- wait
|
||||
- command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
|
||||
name: "stable"
|
||||
timeout_in_minutes: 60
|
||||
artifact_paths: "log-*.txt"
|
||||
- wait
|
||||
- command: "ci/test-stable-perf.sh"
|
||||
name: "stable-perf"
|
||||
timeout_in_minutes: 40
|
||||
artifact_paths: "log-*.txt"
|
||||
agents:
|
||||
- "queue=cuda"
|
||||
- command: "ci/test-bench.sh"
|
||||
name: "bench"
|
||||
timeout_in_minutes: 30
|
||||
- command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-local-cluster.sh"
|
||||
name: "local-cluster"
|
||||
timeout_in_minutes: 45
|
||||
artifact_paths: "log-*.txt"
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
- command: "ci/dependabot-pr.sh"
|
||||
name: "dependabot"
|
||||
timeout_in_minutes: 5
|
||||
if: build.env("GITHUB_USER") == "dependabot-preview[bot]"
|
||||
|
||||
- wait
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
- command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image docs/build.sh"
|
||||
name: "docs"
|
||||
timeout_in_minutes: 5
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Builds a buildkite pipeline based on the environment variables
|
||||
#
|
||||
|
||||
set -e
|
||||
cd "$(dirname "$0")"/../..
|
||||
|
||||
output_file=${1:-/dev/stderr}
|
||||
|
||||
start_pipeline() {
|
||||
echo "# $*" > "$output_file"
|
||||
echo "steps:" >> "$output_file"
|
||||
}
|
||||
|
||||
wait_step() {
|
||||
echo " - wait" >> "$output_file"
|
||||
}
|
||||
|
||||
cat_steps() {
|
||||
cat "$@" >> "$output_file"
|
||||
}
|
||||
|
||||
pull_or_push_steps() {
|
||||
cat_steps ci/buildkite/sanity.yml
|
||||
wait_step
|
||||
|
||||
if ci/affects-files.sh .sh$; then
|
||||
cat_steps ci/buildkite/shellcheck.yml
|
||||
fi
|
||||
wait_step
|
||||
|
||||
docs=false
|
||||
all_tests=false
|
||||
|
||||
if ci/affects-files.sh ^docs/; then
|
||||
docs=true
|
||||
fi
|
||||
|
||||
if ci/affects-files.sh !^docs/ !.md$ !^.buildkite/; then
|
||||
all_tests=true
|
||||
fi
|
||||
|
||||
if $docs; then
|
||||
cat_steps ci/buildkite/docs.yml
|
||||
fi
|
||||
|
||||
if $all_tests; then
|
||||
cat_steps ci/buildkite/all-tests.yml
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
if [[ -n $BUILDKITE_TAG ]]; then
|
||||
start_pipeline "Tag pipeline for $BUILDKITE_TAG"
|
||||
|
||||
buildkite-agent annotate --style info --context release-tag \
|
||||
"https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG"
|
||||
|
||||
# Jump directly to the secondary build to publish release artifacts quickly
|
||||
cat_steps ci/buildkite/trigger-secondary.yml
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
|
||||
start_pipeline "Pull request pipeline for $BUILDKITE_BRANCH"
|
||||
|
||||
# Add helpful link back to the corresponding Github Pull Request
|
||||
buildkite-agent annotate --style info --context pr-backlink \
|
||||
"Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
|
||||
|
||||
cat_steps ci/buildkite/dependabot-pr.yml
|
||||
pull_or_push_steps
|
||||
exit 0
|
||||
fi
|
||||
|
||||
start_pipeline "Push pipeline for $BUILDKITE_BRANCH"
|
||||
pull_or_push_steps
|
||||
wait_step
|
||||
cat_steps ci/buildkite/trigger-secondary.yml
|
||||
exit 0
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
- command: "ci/test-sanity.sh"
|
||||
name: "sanity"
|
||||
timeout_in_minutes: 5
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
- command: "ci/shellcheck.sh"
|
||||
name: "shellcheck"
|
||||
timeout_in_minutes: 5
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
- trigger: "solana-secondary"
|
||||
branches: "!pull/*"
|
||||
async: true
|
||||
build:
|
||||
message: "${BUILDKITE_MESSAGE}"
|
||||
commit: "${BUILDKITE_COMMIT}"
|
||||
branch: "${BUILDKITE_BRANCH}"
|
||||
env:
|
||||
TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"
|
||||
|
|
@ -2,25 +2,6 @@
|
|||
set -e
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
annotate() {
|
||||
${BUILDKITE:-false} && {
|
||||
buildkite-agent annotate "$@"
|
||||
}
|
||||
}
|
||||
|
||||
ci/affects-files.sh \
|
||||
.rs$ \
|
||||
Cargo.lock$ \
|
||||
Cargo.toml$ \
|
||||
^ci/rust-version.sh \
|
||||
^ci/test-bench.sh \
|
||||
|| {
|
||||
annotate --style info --context test-bench \
|
||||
"Bench skipped as no .rs files were modified"
|
||||
exit 0
|
||||
}
|
||||
|
||||
|
||||
source ci/_
|
||||
source ci/upload-ci-artifact.sh
|
||||
|
||||
|
|
|
|||
|
|
@ -8,19 +8,6 @@ annotate() {
|
|||
}
|
||||
}
|
||||
|
||||
ci/affects-files.sh \
|
||||
.rs$ \
|
||||
Cargo.lock$ \
|
||||
Cargo.toml$ \
|
||||
^ci/rust-version.sh \
|
||||
^ci/test-coverage.sh \
|
||||
^scripts/coverage.sh \
|
||||
|| {
|
||||
annotate --style info --context test-coverage \
|
||||
"Coverage skipped as no .rs files were modified"
|
||||
exit 0
|
||||
}
|
||||
|
||||
source ci/upload-ci-artifact.sh
|
||||
source scripts/ulimit-n.sh
|
||||
|
||||
|
|
|
|||
|
|
@ -41,24 +41,6 @@ test-stable)
|
|||
_ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
|
||||
;;
|
||||
test-stable-perf)
|
||||
ci/affects-files.sh \
|
||||
.rs$ \
|
||||
Cargo.lock$ \
|
||||
Cargo.toml$ \
|
||||
^ci/rust-version.sh \
|
||||
^ci/test-stable-perf.sh \
|
||||
^ci/test-stable.sh \
|
||||
^ci/test-local-cluster.sh \
|
||||
^core/build.rs \
|
||||
^fetch-perf-libs.sh \
|
||||
^programs/ \
|
||||
^sdk/ \
|
||||
|| {
|
||||
annotate --style info \
|
||||
"Skipped $testName as no relevant files were modified"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# BPF program tests
|
||||
_ make -C programs/bpf/c tests
|
||||
_ cargo +"$rust_stable" test \
|
||||
|
|
@ -85,22 +67,22 @@ test-stable-perf)
|
|||
_ cargo +"$rust_stable" test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
|
||||
;;
|
||||
test-move)
|
||||
ci/affects-files.sh \
|
||||
Cargo.lock$ \
|
||||
Cargo.toml$ \
|
||||
^ci/rust-version.sh \
|
||||
^ci/test-stable.sh \
|
||||
^ci/test-move.sh \
|
||||
^programs/move_loader \
|
||||
^programs/librapay \
|
||||
^logger/ \
|
||||
^runtime/ \
|
||||
^sdk/ \
|
||||
|| {
|
||||
annotate --style info \
|
||||
"Skipped $testName as no relevant files were modified"
|
||||
exit 0
|
||||
}
|
||||
#ci/affects-files.sh \
|
||||
# Cargo.lock$ \
|
||||
# Cargo.toml$ \
|
||||
# ^ci/rust-version.sh \
|
||||
# ^ci/test-stable.sh \
|
||||
# ^ci/test-move.sh \
|
||||
# ^programs/move_loader \
|
||||
# ^programs/librapay \
|
||||
# ^logger/ \
|
||||
# ^runtime/ \
|
||||
# ^sdk/ \
|
||||
#|| {
|
||||
# annotate --style info \
|
||||
# "Skipped $testName as no relevant files were modified"
|
||||
# exit 0
|
||||
#}
|
||||
_ cargo +"$rust_stable" test --manifest-path programs/move_loader/Cargo.toml ${V:+--verbose} -- --nocapture
|
||||
_ cargo +"$rust_stable" test --manifest-path programs/librapay/Cargo.toml ${V:+--verbose} -- --nocapture
|
||||
exit 0
|
||||
|
|
|
|||
Loading…
Reference in New Issue