diff --git a/.buildkite/scripts/build-downstream-projects.sh b/.buildkite/scripts/build-downstream-projects.sh new file mode 100755 index 000000000..d7c5babcb --- /dev/null +++ b/.buildkite/scripts/build-downstream-projects.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +here=$(dirname "$0") + +# shellcheck source=.buildkite/scripts/common.sh +source "$here"/common.sh + +agent="${1-solana}" + +group "downstream projects" \ + '{ "name": "spl", "command": "./ci/downstream-projects/run-spl.sh", "timeout_in_minutes": 30, "agent": "'"$agent"'" }' \ + '{ "name": "openbook-dex", "command": "./ci/downstream-projects/run-openbook-dex.sh", "timeout_in_minutes": 30, "agent": "'"$agent"'" }' \ + '{ "name": "example-helloworld", "command": "./ci/downstream-projects/run-example-helloworld.sh", "timeout_in_minutes": 30, "agent": "'"$agent"'" }' diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh new file mode 100644 index 000000000..bc2dc807a --- /dev/null +++ b/.buildkite/scripts/common.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +export INDENT_LEVEL=2 + +indent() { + sed "s/^/$(printf ' %.0s' $(seq 1 $INDENT_LEVEL))/" +} + +group() { + cat <> "$output_file" <<"EOF" - - command: "scripts/build-downstream-projects.sh" - name: "downstream-projects" - timeout_in_minutes: 35 - agents: - queue: "solana" -EOF + .buildkite/scripts/build-downstream-projects.sh >> "$output_file" else annotate --style info \ "downstream-projects skipped as no relevant files were modified" diff --git a/ci/buildkite-solana-private.sh b/ci/buildkite-solana-private.sh index bc9ac4f04..523ec8011 100644 --- a/ci/buildkite-solana-private.sh +++ b/ci/buildkite-solana-private.sh @@ -236,15 +236,10 @@ EOF ^fetch-perf-libs.sh \ ^programs/ \ ^sdk/ \ - ^scripts/build-downstream-projects.sh \ + ^ci/downstream-projects \ + .buildkite/scripts/build-downstream-projects.sh \ ; then - cat >> "$output_file" <<"EOF" - - command: "scripts/build-downstream-projects.sh" - name: "downstream-projects" - timeout_in_minutes: 40 - agents: - queue: "sol-private" -EOF + .buildkite/scripts/build-downstream-projects.sh sol-private >> "$output_file" else annotate --style info \ "downstream-projects skipped as no relevant files were modified" diff --git a/ci/downstream-projects/common.sh b/ci/downstream-projects/common.sh new file mode 100755 index 000000000..54a7c8c5f --- /dev/null +++ b/ci/downstream-projects/common.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -e + +source ci/_ +source ci/semver_bash/semver.sh +source scripts/patch-crates.sh +source scripts/read-cargo-variable.sh + +SOLANA_VER=$(readCargoVariable version Cargo.toml) +export SOLANA_VER +export SOLANA_DIR=$PWD +export CARGO="$SOLANA_DIR"/cargo +export CARGO_BUILD_SBF="$SOLANA_DIR"/cargo-build-sbf +export CARGO_TEST_SBF="$SOLANA_DIR"/cargo-test-sbf + +mkdir -p target/downstream-projects +cd target/downstream-projects diff --git a/ci/downstream-projects/func-example-helloworld.sh b/ci/downstream-projects/func-example-helloworld.sh new file mode 100755 index 000000000..34a96438e --- /dev/null +++ b/ci/downstream-projects/func-example-helloworld.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +example_helloworld() { + ( + set -x + rm -rf example-helloworld + git clone https://github.com/solana-labs/example-helloworld.git + # copy toolchain file to use solana's rust version + cp "$SOLANA_DIR"/rust-toolchain.toml example-helloworld/ + cd example-helloworld || exit 1 + + update_solana_dependencies src/program-rust "$SOLANA_VER" + patch_crates_io_solana src/program-rust/Cargo.toml "$SOLANA_DIR" + echo "[workspace]" >> src/program-rust/Cargo.toml + + $CARGO_BUILD_SBF \ + --manifest-path src/program-rust/Cargo.toml + + # TODO: Build src/program-c/... + ) +} diff --git a/ci/downstream-projects/func-openbook-dex.sh b/ci/downstream-projects/func-openbook-dex.sh new file mode 100755 index 000000000..71adddf28 --- /dev/null +++ b/ci/downstream-projects/func-openbook-dex.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +openbook_dex() { + ( + set -x + rm -rf openbook-dex + git clone https://github.com/openbook-dex/program.git openbook-dex + # copy toolchain file to use solana's rust version + cp "$SOLANA_DIR"/rust-toolchain.toml openbook-dex/ + cd openbook-dex || exit 1 + + update_solana_dependencies . "$SOLANA_VER" + patch_crates_io_solana Cargo.toml "$SOLANA_DIR" + cat >> Cargo.toml <> dex/Cargo.toml <=<~]*(.*)\"/\1/p' <"token/program/Cargo.toml") + echo "used solana version: $project_used_solana_version" + if semverGT "$project_used_solana_version" "$SOLANA_VER"; then + echo "skip" + return + fi + + ./patch.crates-io.sh "$SOLANA_DIR" + + for program in "${PROGRAMS[@]}"; do + $CARGO_BUILD_SBF --manifest-path "$program"/Cargo.toml + done + + # TODO better: `build.rs` for spl-token-cli doesn't seem to properly build + # the required programs to run the tests, so instead we run the tests + # after we know programs have been built + cargo build + cargo test + ) +} diff --git a/ci/downstream-projects/run-all.sh b/ci/downstream-projects/run-all.sh new file mode 100755 index 000000000..5f78e91e6 --- /dev/null +++ b/ci/downstream-projects/run-all.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -e + +here="$(dirname "$0")" + +#shellcheck source=ci/downstream-projects/func-example-helloworld.sh +source "$here"/func-example-helloworld.sh + +#shellcheck source=ci/downstream-projects/func-spl.sh +source "$here"/func-spl.sh + +#shellcheck source=ci/downstream-projects/func-openbook-dex.sh +source "$here"/func-openbook-dex.sh + +#shellcheck source=ci/downstream-projects/common.sh +source "$here"/common.sh + +_ example_helloworld +_ spl +_ openbook_dex diff --git a/ci/downstream-projects/run-example-helloworld.sh b/ci/downstream-projects/run-example-helloworld.sh new file mode 100755 index 000000000..00d0db5dd --- /dev/null +++ b/ci/downstream-projects/run-example-helloworld.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +here="$(dirname "$0")" + +#shellcheck source=ci/downstream-projects/func-example-helloworld.sh +source "$here"/func-example-helloworld.sh + +#shellcheck source=ci/downstream-projects/common.sh +source "$here"/common.sh + +_ example_helloworld diff --git a/ci/downstream-projects/run-openbook-dex.sh b/ci/downstream-projects/run-openbook-dex.sh new file mode 100755 index 000000000..d79df5464 --- /dev/null +++ b/ci/downstream-projects/run-openbook-dex.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +here="$(dirname "$0")" + +#shellcheck source=ci/downstream-projects/func-openbook-dex.sh +source "$here"/func-openbook-dex.sh + +#shellcheck source=ci/downstream-projects/common.sh +source "$here"/common.sh + +_ openbook_dex diff --git a/ci/downstream-projects/run-spl.sh b/ci/downstream-projects/run-spl.sh new file mode 100755 index 000000000..543dca928 --- /dev/null +++ b/ci/downstream-projects/run-spl.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +here="$(dirname "$0")" + +#shellcheck source=ci/downstream-projects/func-spl.sh +source "$here"/func-spl.sh + +#shellcheck source=ci/downstream-projects/common.sh +source "$here"/common.sh + +_ spl diff --git a/ci/test-downstream-builds.sh b/ci/test-downstream-builds.sh index 0a45db80d..9ddba8755 100755 --- a/ci/test-downstream-builds.sh +++ b/ci/test-downstream-builds.sh @@ -6,4 +6,4 @@ export CI_LOCAL_RUN=true set -ex -scripts/build-downstream-projects.sh +ci/downstream-projects/run-all.sh diff --git a/scripts/build-downstream-projects.sh b/scripts/build-downstream-projects.sh deleted file mode 100755 index a3f2936fa..000000000 --- a/scripts/build-downstream-projects.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -# -# Builds known downstream projects against local solana source -# - -set -e -cd "$(dirname "$0")"/.. -source ci/_ -source ci/semver_bash/semver.sh -source scripts/patch-crates.sh -source scripts/read-cargo-variable.sh - -solana_ver=$(readCargoVariable version Cargo.toml) -solana_dir=$PWD -cargo_build_sbf="$solana_dir"/cargo-build-sbf -cargo_test_sbf="$solana_dir"/cargo-test-sbf - -mkdir -p target/downstream-projects -cd target/downstream-projects - -example_helloworld() { - ( - set -x - rm -rf example-helloworld - git clone https://github.com/solana-labs/example-helloworld.git - # copy toolchain file to use solana's rust version - cp "$solana_dir"/rust-toolchain.toml example-helloworld/ - cd example-helloworld - - update_solana_dependencies src/program-rust "$solana_ver" - patch_crates_io_solana src/program-rust/Cargo.toml "$solana_dir" - echo "[workspace]" >> src/program-rust/Cargo.toml - - $cargo_build_sbf \ - --manifest-path src/program-rust/Cargo.toml - - # TODO: Build src/program-c/... - ) -} - -spl() { - ( - # Mind the order! - PROGRAMS=( - instruction-padding/program - token/program - token/program-2022 - token/program-2022-test - associated-token-account/program - token-upgrade/program - feature-proposal/program - governance/addin-mock/program - governance/program - memo/program - name-service/program - stake-pool/program - ) - set -x - rm -rf spl - git clone https://github.com/solana-labs/solana-program-library.git spl - # copy toolchain file to use solana's rust version - cp "$solana_dir"/rust-toolchain.toml spl/ - cd spl - - project_used_solana_version=$(sed -nE 's/solana-sdk = \"[>=<~]*(.*)\"/\1/p' <"token/program/Cargo.toml") - echo "used solana version: $project_used_solana_version" - if semverGT "$project_used_solana_version" "$solana_ver"; then - echo "skip" - return - fi - - ./patch.crates-io.sh "$solana_dir" - - for program in "${PROGRAMS[@]}"; do - $cargo_test_sbf --manifest-path "$program"/Cargo.toml - done - - # TODO better: `build.rs` for spl-token-cli doesn't seem to properly build - # the required programs to run the tests, so instead we run the tests - # after we know programs have been built - cargo build - cargo test - ) -} - -openbook_dex() { - ( - set -x - rm -rf openbook-dex - git clone https://github.com/openbook-dex/program.git openbook-dex - # copy toolchain file to use solana's rust version - cp "$solana_dir"/rust-toolchain.toml openbook-dex/ - cd openbook-dex - - update_solana_dependencies . "$solana_ver" - patch_crates_io_solana Cargo.toml "$solana_dir" - cat >> Cargo.toml <> dex/Cargo.toml <