ci: reorg downstream projects (#30463)

reorg downstream projects

.buildkite/scripts/build-downstream-projects.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+# shellcheck source=.buildkite/scripts/common.sh
+source "$here"/common.sh
+
+agent="${1-solana}"
+
+group "downstream projects" \
+  '{ "name": "spl", "command": "./ci/downstream-projects/run-spl.sh", "timeout_in_minutes": 30, "agent": "'"$agent"'" }' \
+  '{ "name": "openbook-dex", "command": "./ci/downstream-projects/run-openbook-dex.sh", "timeout_in_minutes": 30, "agent": "'"$agent"'" }' \
+  '{ "name": "example-helloworld", "command": "./ci/downstream-projects/run-example-helloworld.sh", "timeout_in_minutes": 30, "agent": "'"$agent"'" }'

.buildkite/scripts/common.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+export INDENT_LEVEL=2
+
+indent() {
+  sed "s/^/$(printf ' %.0s' $(seq 1 $INDENT_LEVEL))/"
+}
+
+group() {
+  cat <<EOF | indent
+- group: "$1"
+  steps:
+EOF
+  shift
+
+  INDENT_LEVEL=$((INDENT_LEVEL + 4))
+  for params in "$@"; do
+    step "$params"
+  done
+  INDENT_LEVEL=$((INDENT_LEVEL - 4))
+}
+
+step() {
+  local params="$1"
+
+  local name
+  name="$(echo "$params" | jq -r '.name')"
+
+  local command
+  command="$(echo "$params" | jq -r '.command')"
+
+  local timeout_in_minutes
+  timeout_in_minutes="$(echo "$params" | jq -r '.timeout_in_minutes')"
+
+  local agent
+  agent="$(echo "$params" | jq -r '.agent')"
+
+  cat <<EOF | indent
+- name: "$name"
+  command: "$command"
+  timeout_in_minutes: $timeout_in_minutes
+  agents:
+    queue: "$agent"
+EOF
+}

ci/buildkite-pipeline.sh

@@ -245,19 +245,14 @@ EOF
              ^fetch-perf-libs.sh \
              ^programs/ \
              ^sdk/ \
-             ^scripts/build-downstream-projects.sh \
              cargo-build-bpf$ \
              cargo-test-bpf$ \
              cargo-build-sbf$ \
              cargo-test-sbf$ \
+             ^ci/downstream-projects \
+             .buildkite/scripts/build-downstream-projects.sh \
       ; then
-    cat >> "$output_file" <<"EOF"
-  - command: "scripts/build-downstream-projects.sh"
-    name: "downstream-projects"
-    timeout_in_minutes: 35
-    agents:
-      queue: "solana"
-EOF
+    .buildkite/scripts/build-downstream-projects.sh >> "$output_file"
   else
     annotate --style info \
       "downstream-projects skipped as no relevant files were modified"

ci/buildkite-solana-private.sh

@@ -236,15 +236,10 @@ EOF
              ^fetch-perf-libs.sh \
              ^programs/ \
              ^sdk/ \
-             ^scripts/build-downstream-projects.sh \
+             ^ci/downstream-projects \
+             .buildkite/scripts/build-downstream-projects.sh \
       ; then
-    cat >> "$output_file" <<"EOF"
-  - command: "scripts/build-downstream-projects.sh"
-    name: "downstream-projects"
-    timeout_in_minutes: 40
-    agents:
-      queue: "sol-private"
-EOF
+    .buildkite/scripts/build-downstream-projects.sh sol-private >> "$output_file"
   else
     annotate --style info \
       "downstream-projects skipped as no relevant files were modified"

ci/downstream-projects/common.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+source ci/_
+source ci/semver_bash/semver.sh
+source scripts/patch-crates.sh
+source scripts/read-cargo-variable.sh
+
+SOLANA_VER=$(readCargoVariable version Cargo.toml)
+export SOLANA_VER
+export SOLANA_DIR=$PWD
+export CARGO="$SOLANA_DIR"/cargo
+export CARGO_BUILD_SBF="$SOLANA_DIR"/cargo-build-sbf
+export CARGO_TEST_SBF="$SOLANA_DIR"/cargo-test-sbf
+
+mkdir -p target/downstream-projects
+cd target/downstream-projects

ci/downstream-projects/func-example-helloworld.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+example_helloworld() {
+  (
+    set -x
+    rm -rf example-helloworld
+    git clone https://github.com/solana-labs/example-helloworld.git
+    # copy toolchain file to use solana's rust version
+    cp "$SOLANA_DIR"/rust-toolchain.toml example-helloworld/
+    cd example-helloworld || exit 1
+
+    update_solana_dependencies src/program-rust "$SOLANA_VER"
+    patch_crates_io_solana src/program-rust/Cargo.toml "$SOLANA_DIR"
+    echo "[workspace]" >> src/program-rust/Cargo.toml
+
+    $CARGO_BUILD_SBF \
+      --manifest-path src/program-rust/Cargo.toml
+
+    # TODO: Build src/program-c/...
+  )
+}

ci/downstream-projects/func-openbook-dex.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+openbook_dex() {
+  (
+    set -x
+    rm -rf openbook-dex
+    git clone https://github.com/openbook-dex/program.git openbook-dex
+    # copy toolchain file to use solana's rust version
+    cp "$SOLANA_DIR"/rust-toolchain.toml openbook-dex/
+    cd openbook-dex || exit 1
+
+    update_solana_dependencies . "$SOLANA_VER"
+    patch_crates_io_solana Cargo.toml "$SOLANA_DIR"
+    cat >> Cargo.toml <<EOF
+anchor-lang = { git = "https://github.com/coral-xyz/anchor.git", branch = "master" }
+EOF
+    patch_crates_io_solana dex/Cargo.toml "$SOLANA_DIR"
+    cat >> dex/Cargo.toml <<EOF
+anchor-lang = { git = "https://github.com/coral-xyz/anchor.git", branch = "master" }
+[workspace]
+exclude = [
+    "crank",
+    "permissioned",
+]
+EOF
+    cargo build
+
+    $CARGO_BUILD_SBF \
+      --manifest-path dex/Cargo.toml --no-default-features --features program
+
+    cargo test \
+      --manifest-path dex/Cargo.toml --no-default-features --features program
+  )
+}

ci/downstream-projects/func-spl.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+spl() {
+  (
+    # Mind the order!
+    PROGRAMS=(
+      instruction-padding/program
+      token/program
+      token/program-2022
+      token/program-2022-test
+      associated-token-account/program
+      token-upgrade/program
+      feature-proposal/program
+      governance/addin-mock/program
+      governance/program
+      memo/program
+      name-service/program
+      stake-pool/program
+    )
+    set -x
+    rm -rf spl
+    git clone https://github.com/solana-labs/solana-program-library.git spl
+    # copy toolchain file to use solana's rust version
+    cp "$SOLANA_DIR"/rust-toolchain.toml spl/
+    cd spl || exit 1
+
+    project_used_solana_version=$(sed -nE 's/solana-sdk = \"[>=<~]*(.*)\"/\1/p' <"token/program/Cargo.toml")
+    echo "used solana version: $project_used_solana_version"
+    if semverGT "$project_used_solana_version" "$SOLANA_VER"; then
+      echo "skip"
+      return
+    fi
+
+    ./patch.crates-io.sh "$SOLANA_DIR"
+
+    for program in "${PROGRAMS[@]}"; do
+      $CARGO_BUILD_SBF --manifest-path "$program"/Cargo.toml
+    done
+
+    # TODO better: `build.rs` for spl-token-cli doesn't seem to properly build
+    # the required programs to run the tests, so instead we run the tests
+    # after we know programs have been built
+    cargo build
+    cargo test
+  )
+}

ci/downstream-projects/run-all.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+
+here="$(dirname "$0")"
+
+#shellcheck source=ci/downstream-projects/func-example-helloworld.sh
+source "$here"/func-example-helloworld.sh
+
+#shellcheck source=ci/downstream-projects/func-spl.sh
+source "$here"/func-spl.sh
+
+#shellcheck source=ci/downstream-projects/func-openbook-dex.sh
+source "$here"/func-openbook-dex.sh
+
+#shellcheck source=ci/downstream-projects/common.sh
+source "$here"/common.sh
+
+_ example_helloworld
+_ spl
+_ openbook_dex

ci/downstream-projects/run-example-helloworld.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+here="$(dirname "$0")"
+
+#shellcheck source=ci/downstream-projects/func-example-helloworld.sh
+source "$here"/func-example-helloworld.sh
+
+#shellcheck source=ci/downstream-projects/common.sh
+source "$here"/common.sh
+
+_ example_helloworld

ci/downstream-projects/run-openbook-dex.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+here="$(dirname "$0")"
+
+#shellcheck source=ci/downstream-projects/func-openbook-dex.sh
+source "$here"/func-openbook-dex.sh
+
+#shellcheck source=ci/downstream-projects/common.sh
+source "$here"/common.sh
+
+_ openbook_dex

ci/downstream-projects/run-spl.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+here="$(dirname "$0")"
+
+#shellcheck source=ci/downstream-projects/func-spl.sh
+source "$here"/func-spl.sh
+
+#shellcheck source=ci/downstream-projects/common.sh
+source "$here"/common.sh
+
+_ spl

ci/test-downstream-builds.sh

@@ -6,4 +6,4 @@ export CI_LOCAL_RUN=true
 
 set -ex
 
-scripts/build-downstream-projects.sh
+ci/downstream-projects/run-all.sh

scripts/build-downstream-projects.sh

@@ -1,121 +0,0 @@
-#!/usr/bin/env bash
-#
-# Builds known downstream projects against local solana source
-#
-
-set -e
-cd "$(dirname "$0")"/..
-source ci/_
-source ci/semver_bash/semver.sh
-source scripts/patch-crates.sh
-source scripts/read-cargo-variable.sh
-
-solana_ver=$(readCargoVariable version Cargo.toml)
-solana_dir=$PWD
-cargo_build_sbf="$solana_dir"/cargo-build-sbf
-cargo_test_sbf="$solana_dir"/cargo-test-sbf
-
-mkdir -p target/downstream-projects
-cd target/downstream-projects
-
-example_helloworld() {
-  (
-    set -x
-    rm -rf example-helloworld
-    git clone https://github.com/solana-labs/example-helloworld.git
-    # copy toolchain file to use solana's rust version
-    cp "$solana_dir"/rust-toolchain.toml example-helloworld/
-    cd example-helloworld
-
-    update_solana_dependencies src/program-rust "$solana_ver"
-    patch_crates_io_solana src/program-rust/Cargo.toml "$solana_dir"
-    echo "[workspace]" >> src/program-rust/Cargo.toml
-
-    $cargo_build_sbf \
-      --manifest-path src/program-rust/Cargo.toml
-
-    # TODO: Build src/program-c/...
-  )
-}
-
-spl() {
-  (
-    # Mind the order!
-    PROGRAMS=(
-      instruction-padding/program
-      token/program
-      token/program-2022
-      token/program-2022-test
-      associated-token-account/program
-      token-upgrade/program
-      feature-proposal/program
-      governance/addin-mock/program
-      governance/program
-      memo/program
-      name-service/program
-      stake-pool/program
-    )
-    set -x
-    rm -rf spl
-    git clone https://github.com/solana-labs/solana-program-library.git spl
-    # copy toolchain file to use solana's rust version
-    cp "$solana_dir"/rust-toolchain.toml spl/
-    cd spl
-
-    project_used_solana_version=$(sed -nE 's/solana-sdk = \"[>=<~]*(.*)\"/\1/p' <"token/program/Cargo.toml")
-    echo "used solana version: $project_used_solana_version"
-    if semverGT "$project_used_solana_version" "$solana_ver"; then
-      echo "skip"
-      return
-    fi
-
-    ./patch.crates-io.sh "$solana_dir"
-
-    for program in "${PROGRAMS[@]}"; do
-      $cargo_test_sbf --manifest-path "$program"/Cargo.toml
-    done
-
-    # TODO better: `build.rs` for spl-token-cli doesn't seem to properly build
-    # the required programs to run the tests, so instead we run the tests
-    # after we know programs have been built
-    cargo build
-    cargo test
-  )
-}
-
-openbook_dex() {
-  (
-    set -x
-    rm -rf openbook-dex
-    git clone https://github.com/openbook-dex/program.git openbook-dex
-    # copy toolchain file to use solana's rust version
-    cp "$solana_dir"/rust-toolchain.toml openbook-dex/
-    cd openbook-dex
-
-    update_solana_dependencies . "$solana_ver"
-    patch_crates_io_solana Cargo.toml "$solana_dir"
-    cat >> Cargo.toml <<EOF
-anchor-lang = { git = "https://github.com/coral-xyz/anchor.git", branch = "master" }
-EOF
-    patch_crates_io_solana dex/Cargo.toml "$solana_dir"
-    cat >> dex/Cargo.toml <<EOF
-anchor-lang = { git = "https://github.com/coral-xyz/anchor.git", branch = "master" }
-[workspace]
-exclude = [
-    "crank",
-    "permissioned",
-]
-EOF
-    cargo build
-
-    $cargo_build_sbf \
-      --manifest-path dex/Cargo.toml --no-default-features --features program
-
-    cargo test \
-      --manifest-path dex/Cargo.toml --no-default-features --features program
-  )
-}
-
-_ example_helloworld
-_ spl
-_ openbook_dex