ci: run stable tests partially (#31151)

* ci: run stable tests partially

* ci: improve uploading test result

* use buildkite's retry

* test common

* fix indent issue

* extract assert eq

* use diff to print test error

* pretty format

* test build stable steps

* add test all function

* lint

* lint

* keep dollar sign

* switch got and want's diff direction

* use 'EOF' to make code more readable

* rename EMPTY_LINE to DELETE_THIS_LINE

* rename N => INDEX

* rename M => LIMIT

* check variable in a more elegant way

* wording

* more explain for code

.buildkite/hooks/post-command

@@ -2,6 +2,9 @@
 
 source ci/upload-ci-artifact.sh
 
+#shellcheck source=ci/common/shared-functions.sh
+source ci/common/shared-functions.sh
+
 #
 # Add job_stats data point
 #
@@ -32,19 +35,45 @@ else
   fi
 
   if [[ -n $BUILDKITE && -f "results.json" ]]; then
+    if need_to_upload_test_result; then
+      if [[ -f "results.json" ]]; then
         # extract lines which start with '{'
-    awk '/{.*/' results.json > sanitized-results.json
+        awk '/{.*/' results.json >sanitized-results.json
 
         echo "~~~ Uploading test results to Buildkite Analytics"
-    buildkite-test-collector < sanitized-results.json
+        buildkite-test-collector <sanitized-results.json
 
         echo "~~~ Uploading test results to Datadog"
-    cargo2junit > results.xml < sanitized-results.json || true
+        cargo2junit >results.xml <sanitized-results.json || true
         if [[ -f "results.xml" ]]; then
           datadog-ci junit upload --service solana results.xml
         fi
       fi
 
+      if [[ -f "target/nextest/ci/junit.xml" ]]; then
+        echo "~~~ Uploading test results to Buildkite Analytics"
+        curl \
+          -X POST \
+          -H "Authorization: Token token=\"$BUILDKITE_ANALYTICS_TOKEN\"" \
+          -F "data=@target/nextest/ci/junit.xml" \
+          -F "format=junit" \
+          -F "run_env[CI]=buildkite" \
+          -F "run_env[key]=$BUILDKITE_BUILD_ID" \
+          -F "run_env[url]=$BUILDKITE_BUILD_URL" \
+          -F "run_env[branch]=$BUILDKITE_BRANCH" \
+          -F "run_env[commit_sha]=$BUILDKITE_COMMIT" \
+          -F "run_env[number]=$BUILDKITE_BUILD_NUMBER" \
+          -F "run_env[job_id]=$BUILDKITE_JOB_ID" \
+          -F "run_env[message]=$BUILDKITE_MESSAGE" \
+          https://analytics-api.buildkite.com/v1/uploads
+        echo # add a break line for previous command
+
+        echo "~~~ Uploading test results to Datadog"
+        datadog-ci junit upload --service solana target/nextest/ci/junit.xml
+      fi
+    fi
+  fi
+
   point_tags="pipeline=$BUILDKITE_PIPELINE_SLUG,job=$CI_LABEL,pr=$PR,success=$SUCCESS"
   point_tags="${point_tags// /\\ }"  # Escape spaces
 

.buildkite/scripts/build-stable.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -e
+here=$(dirname "$0")
+
+# shellcheck source=.buildkite/scripts/common.sh
+source "$here"/common.sh
+
+agent="${1-solana}"
+
+partitions=$(
+  cat <<EOF
+{
+  "name": "partitions",
+  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-partition.sh",
+  "timeout_in_minutes": 30,
+  "agent": "$agent",
+  "parallelism": 3,
+  "retry": 3
+}
+EOF
+)
+
+localnet=$(
+  cat <<EOF
+{
+  "name": "localnet",
+  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-localnet.sh",
+  "timeout_in_minutes": 30,
+  "agent": "$agent"
+}
+EOF
+)
+
+# shellcheck disable=SC2016
+group "stable" "$partitions" "$localnet"

.buildkite/scripts/build-stable.test.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e
+here=$(dirname "$0")
+
+# shellcheck source=.buildkite/scripts/func-assert-eq.sh
+source "$here"/func-assert-eq.sh
+
+want=$(
+  cat <<'EOF'
+  - group: "stable"
+    steps:
+      - name: "partitions"
+        command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/stable/run-partition.sh"
+        timeout_in_minutes: 30
+        agents:
+          queue: "solana"
+        parallelism: 3
+        retry:
+          automatic:
+            - limit: 3
+      - name: "localnet"
+        command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/stable/run-localnet.sh"
+        timeout_in_minutes: 30
+        agents:
+          queue: "solana"
+EOF
+)
+
+# shellcheck source=.buildkite/scripts/build-stable.sh
+got=$(source "$here"/build-stable.sh)
+
+assert_eq "test build stable steps" "$want" "$got"

.buildkite/scripts/common.sh

@@ -3,7 +3,8 @@
 export INDENT_LEVEL=2
 
 indent() {
-  sed "s/^/$(printf ' %.0s' $(seq 1 $INDENT_LEVEL))/"
+  local indent=${1:-"$INDENT_LEVEL"}
+  sed "s/^/$(printf ' %.0s' $(seq 1 "$indent"))/"
 }
 
 group() {
@@ -35,11 +36,37 @@ step() {
   local agent
   agent="$(echo "$params" | jq -r '.agent')"
 
-  cat <<EOF | indent
+  local parallelism
+  parallelism="$(echo "$params" | jq -r '.parallelism')"
+  maybe_parallelism="DELETE_THIS_LINE"
+  if [ "$parallelism" != "null" ]; then
+    maybe_parallelism=$(
+      cat <<EOF | indent 2
+parallelism: $parallelism
+EOF
+    )
+  fi
+
+  local retry
+  retry="$(echo "$params" | jq -r '.retry')"
+  maybe_retry="DELETE_THIS_LINE"
+  if [ "$retry" != "null" ]; then
+    maybe_retry=$(
+      cat <<EOF | indent 2
+retry:
+  automatic:
+    - limit: $retry
+EOF
+    )
+  fi
+
+  cat <<EOF | indent | sed '/DELETE_THIS_LINE/d'
 - name: "$name"
   command: "$command"
   timeout_in_minutes: $timeout_in_minutes
   agents:
     queue: "$agent"
+$maybe_parallelism
+$maybe_retry
 EOF
 }

.buildkite/scripts/common.test.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+set -e
+here=$(dirname "$0")
+
+# shellcheck source=.buildkite/scripts/func-assert-eq.sh
+source "$here"/func-assert-eq.sh
+
+# shellcheck source=.buildkite/scripts/common.sh
+source "$here"/common.sh
+
+(
+  want=$(
+    cat <<EOF | indent
+- name: "test"
+  command: "start.sh"
+  timeout_in_minutes: 10
+  agents:
+    queue: "agent"
+EOF
+  )
+
+  got=$(step '{ "name": "test", "command": "start.sh", "timeout_in_minutes": 10, "agent": "agent"}')
+
+  assert_eq "basic setup" "$want" "$got"
+)
+
+(
+  want=$(
+    cat <<EOF | indent
+- name: "test"
+  command: "start.sh"
+  timeout_in_minutes: 10
+  agents:
+    queue: "agent"
+  parallelism: 3
+EOF
+  )
+
+  got=$(step '{ "name": "test", "command": "start.sh", "timeout_in_minutes": 10, "agent": "agent", "parallelism": 3}')
+
+  assert_eq "basic setup + parallelism" "$want" "$got"
+)
+
+(
+  want=$(
+    cat <<EOF | indent
+- name: "test"
+  command: "start.sh"
+  timeout_in_minutes: 10
+  agents:
+    queue: "agent"
+  retry:
+    automatic:
+      - limit: 3
+EOF
+  )
+
+  got=$(step '{ "name": "test", "command": "start.sh", "timeout_in_minutes": 10, "agent": "agent", "retry": 3}')
+
+  assert_eq "basic setup + retry" "$want" "$got"
+)
+
+(
+  want=$(
+    cat <<EOF | indent
+- name: "test"
+  command: "start.sh"
+  timeout_in_minutes: 10
+  agents:
+    queue: "agent"
+  parallelism: 3
+  retry:
+    automatic:
+      - limit: 3
+EOF
+  )
+
+  got=$(step '{ "name": "test", "command": "start.sh", "timeout_in_minutes": 10, "agent": "agent", "parallelism": 3, "retry": 3}')
+
+  assert_eq "basic setup + parallelism + retry" "$want" "$got"
+)

.buildkite/scripts/func-assert-eq.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+assert_eq() {
+  local test_name=$1
+  local want=$2
+  local got=$3
+
+  if [[ "$want" = "$got" ]]; then
+    echo "✅ $test_name"
+  else
+    cat <<EOF
+❌ $test_name
+$(diff -u <(echo "$want") <(echo "$got"))
+EOF
+  fi
+}

.buildkite/scripts/test-all.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+find "$here" -name '*.test.sh' -exec {} \;

ci/buildkite-pipeline-in-disk.sh

@@ -166,7 +166,7 @@ all_test_steps() {
       "Coverage skipped as no .rs files were modified"
   fi
   # Full test suite
-  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 70
+  .buildkite/scripts/build-stable.sh >> "$output_file"
   wait_step
 
   # SBF test suite

ci/buildkite-pipeline.sh

@@ -143,7 +143,7 @@ all_test_steps() {
   wait_step
 
   # Full test suite
-  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 70
+  .buildkite/scripts/build-stable.sh >> "$output_file"
 
   # Docs tests
   if affects \

ci/buildkite-solana-private.sh

@@ -138,7 +138,7 @@ all_test_steps() {
   wait_step
 
   # Full test suite
-  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 70
+  .buildkite/scripts/build-stable.sh >> "$output_file"
 
   # Docs tests
   if affects \

ci/docker-run.sh

@@ -93,6 +93,8 @@ ARGS+=(
   --env BUILDKITE
   --env BUILDKITE_AGENT_ACCESS_TOKEN
   --env BUILDKITE_JOB_ID
+  --env BUILDKITE_PARALLEL_JOB
+  --env BUILDKITE_PARALLEL_JOB_COUNT
   --env CI
   --env CI_BRANCH
   --env CI_BASE_BRANCH

ci/stable/common.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -e
+
+export RUST_BACKTRACE=1
+export RUSTFLAGS="-D warnings"
+
+source ci/_

ci/stable/run-all.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+
+here="$(dirname "$0")"
+
+#shellcheck source=ci/common/shared-functions.sh
+source "$here"/../common/shared-functions.sh
+
+#shellcheck source=ci/common/limit-threads.sh
+source "$here"/../common/limit-threads.sh
+
+#shellcheck source=ci/stable/common.sh
+source "$here"/common.sh
+
+if need_to_generate_test_result; then
+  _ cargo test --jobs "$JOBS" --workspace --tests --verbose -- -Z unstable-options --format json --report-time | tee results.json
+  exit_if_error "${PIPESTATUS[0]}"
+else
+  _ ci/intercept.sh cargo test --jobs "$JOBS" --workspace --tests --verbose -- --nocapture
+fi

ci/stable/run-localnet.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+here="$(dirname "$0")"
+
+export RUST_LOG="solana_metrics=warn,info,$RUST_LOG"
+
+echo --- ci/localnet-sanity.sh
+"$here"/../localnet-sanity.sh -x
+
+echo --- ci/run-sanity.sh
+"$here"/../run-sanity.sh -x

ci/stable/run-partition.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -e
+
+here="$(dirname "$0")"
+
+#shellcheck source=ci/common/shared-functions.sh
+source "$here"/../common/shared-functions.sh
+
+#shellcheck source=ci/common/limit-threads.sh
+source "$here"/../common/limit-threads.sh
+
+#shellcheck source=ci/stable/common.sh
+source "$here"/common.sh
+
+# check partition info
+INDEX=${1:-"$BUILDKITE_PARALLEL_JOB"} # BUILDKITE_PARALLEL_JOB from 0 to (BUILDKITE_PARALLEL_JOB_COUNT - 1)
+: "${INDEX:?}"
+
+# if LIMIT = 3, the valid INDEX is 0~2
+LIMIT=${2:-"$BUILDKITE_PARALLEL_JOB_COUNT"}
+: "${LIMIT:?}"
+
+if [ "$LIMIT" -lt 2 ]; then
+  echo "LIMIT(\$2) should >= 2"
+  exit 1
+fi
+
+if [ ! "$LIMIT" -gt "$INDEX" ]; then
+  echo "LIMIT(\$2) should greater than INDEX(\$1)"
+  exit 1
+fi
+
+DONT_USE_NEXTEST_PACKAGES=(
+  solana-client-test
+  solana-cargo-build-sbf
+  solana-core
+)
+
+if [ "$INDEX" -eq "$((LIMIT - 1))" ]; then
+  ARGS=(
+    --jobs "$JOBS"
+    --tests
+    --verbose
+  )
+  for package in "${DONT_USE_NEXTEST_PACKAGES[@]}"; do
+    ARGS+=(-p "$package")
+  done
+
+  if need_to_generate_test_result; then
+    _ cargo test "${ARGS[@]}" --verbose -- -Z unstable-options --format json --report-time | tee results.json
+    exit_if_error "${PIPESTATUS[0]}"
+  else
+    _ cargo test "${ARGS[@]}"
+  fi
+else
+  ARGS=(
+    --profile ci
+    --config-file ./nextest.toml
+    --workspace
+    --tests
+    --jobs "$JOBS"
+    --partition hash:"$((INDEX + 1))/$((LIMIT - 1))"
+    --verbose
+    --exclude solana-local-cluster
+  )
+  for package in "${DONT_USE_NEXTEST_PACKAGES[@]}"; do
+    ARGS+=(--exclude "$package")
+  done
+
+  _ cargo nextest run "${ARGS[@]}"
+fi

nextest.toml

@@ -0,0 +1,9 @@
+[store]
+dir = "target/nextest"
+
+[profile.ci]
+failure-output = "immediate-final"
+fail-fast = false
+
+[profile.ci.junit]
+path = "junit.xml"