diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index b301e121..d5b527c8 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -5,30 +5,32 @@ on: push: branches: [master] -env: - SOLANA_VERSION: v1.4.4 - jobs: all_github_action_checks: runs-on: ubuntu-latest needs: + - rustfmt - clippy - - list_bpf_programs - - cargo_test - - js_token - - js_token_swap - - js_token_lending - - build_bpf_programs - - excluded_project_tests + - cargo-build-test + - js-test-token + - js-test-token-swap + - js-test-token-lending steps: - run: echo "Done" + rustfmt: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + + - name: Set env vars + run: | + source ci/rust-version.sh + echo "RUST_STABLE=$rust_stable" >> $GITHUB_ENV + - uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: ${{ env.RUST_STABLE }} override: true profile: minimal components: rustfmt @@ -50,22 +52,6 @@ jobs: profile: minimal components: clippy - - name: Install dependencies - run: | - sh -c "$(curl -sSfL https://release.solana.com/$SOLANA_VERSION/install)" - echo "PATH=/home/runner/.local/share/solana/install/active_release/bin:$PATH" >> $GITHUB_ENV - export PATH="/home/runner/.local/share/solana/install/active_release/bin:$PATH" - cargo build-bpf --version - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" - sudo apt-get update - sudo apt-get install -y clang-7 --allow-unauthenticated - sudo apt-get install -y openssl --allow-unauthenticated - sudo apt-get install -y libssl-dev --allow-unauthenticated - sudo apt-get install -y libssl1.1 --allow-unauthenticated - sudo apt-get install -y libudev-dev - clang-7 --version - - uses: actions/cache@v2 with: path: | @@ -76,63 +62,42 @@ jobs: restore-keys: | cargo-clippy- + - name: Install dependencies + run: ./ci/install-build-deps.sh + - name: Run clippy - if: always() uses: actions-rs/cargo@v1 with: command: clippy args: -Zunstable-options --workspace --all-targets -- --deny=warnings - list_bpf_programs: + cargo-build-test: runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.list-bpf-programs.outputs.matrix }} steps: - uses: actions/checkout@v2 - - id: list-bpf-programs + + - name: Set env vars run: | - JSON="{\"include\":[" + source ci/rust-version.sh + echo "RUST_STABLE=$rust_stable" >> $GITHUB_ENV + source ci/solana-version.sh + echo "SOLANA_VERSION=$solana_version" >> $GITHUB_ENV - for Xargo_toml in $(git ls-files -- '*/Xargo.toml'); do - program_dir=$(dirname "$Xargo_toml") - JSONline="{\"path\": \"$program_dir\"}," - if [[ "$JSON" != *"$JSONline"* ]]; then - JSON="$JSON$JSONline" - fi - done - - # Remove last "," and add closing brackets - if [[ $JSON == *, ]]; then - JSON="${JSON%?}" - fi - JSON="$JSON]}" - echo $JSON - - # Set output - echo "::set-output name=matrix::$( echo "$JSON" )" - - build_bpf_programs: - name: Build Program ${{ matrix.path }} - needs: list_bpf_programs - runs-on: ubuntu-latest - strategy: - matrix: ${{fromJson(needs.list_bpf_programs.outputs.matrix)}} - steps: - - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: 1.47.0 # MSRV + toolchain: ${{ env.RUST_STABLE }} override: true profile: minimal + - uses: actions/cache@v2 with: path: | ~/.cargo/registry ~/.cargo/git target - key: cargo-bpf-${{ matrix.path }}-${{ hashFiles('**/Cargo.lock') }} + key: cargo-build-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - cargo-bpf-${{ matrix.path }}- + cargo-build- - uses: actions/cache@v2 with: @@ -146,151 +111,31 @@ jobs: with: path: | ~/.cache - key: solana-${{ env.SOLANA_VERSION }}-nonce-1 + key: solana-${{ env.SOLANA_VERSION }} restore-keys: | - solana-nonce-1 + solana- - name: Install dependencies run: | - cargo install rustfilt || true - sh -c "$(curl -sSfL https://release.solana.com/$SOLANA_VERSION/install)" - echo "PATH=/home/runner/.local/share/solana/install/active_release/bin:$PATH" >> $GITHUB_ENV - export PATH="/home/runner/.local/share/solana/install/active_release/bin:$PATH" - cargo build-bpf --version + ./ci/install-build-deps.sh + ./ci/install-program-deps.sh + echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - - name: Production check - run: | - if [ "${{ matrix.path }}" == "token-swap/program" ]; then - address="SwaPpA9LAaLfeLi3a68M4DjnLqgtticKg6CnyNwgAC8" - SWAP_PROGRAM_OWNER_FEE_ADDRESS="$address" cargo build-bpf --manifest-path=token-swap/program/Cargo.toml --dump --features production - mv spl_token_swap.so spl_token_swap_production.so - fi + - name: Build and test + run: ./ci/cargo-build-test.sh - - name: Build - uses: actions-rs/cargo@v1 - with: - command: build-bpf - args: --manifest-path=${{ matrix.path }}/Cargo.toml --dump - - - name: Upload program for later jobs + - name: Upload programs uses: actions/upload-artifact@v2 with: name: programs path: "*.so" if-no-files-found: error - cargo_test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.47.0 # MSRV - override: true - profile: minimal - - - name: Install dependencies - run: | - sh -c "$(curl -sSfL https://release.solana.com/$SOLANA_VERSION/install)" - echo "PATH=/home/runner/.local/share/solana/install/active_release/bin:$PATH" >> $GITHUB_ENV - export PATH="/home/runner/.local/share/solana/install/active_release/bin:$PATH" - cargo build-bpf --version - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" - sudo apt-get update - sudo apt-get install -y clang-7 --allow-unauthenticated - sudo apt-get install -y openssl --allow-unauthenticated - sudo apt-get install -y libssl-dev --allow-unauthenticated - sudo apt-get install -y libssl1.1 --allow-unauthenticated - sudo apt-get install -y libudev-dev - clang-7 --version - - - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: cargo-test-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - cargo-test- - - - uses: actions/cache@v2 - with: - path: | - ~/.cache - key: solana-${{ env.SOLANA_VERSION }}-nonce-1 - restore-keys: | - solana-nonce-1 - - - name: Run tests - uses: actions-rs/cargo@v1 - with: - command: test - - excluded_project_tests: - needs: build_bpf_programs - name: Test ${{ matrix.path }} - runs-on: ubuntu-latest - - strategy: - matrix: - path: - - themis/client_ristretto - - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - profile: minimal - - - name: Install dependencies - run: | - sh -c "$(curl -sSfL https://release.solana.com/$SOLANA_VERSION/install)" - echo "PATH=/home/runner/.local/share/solana/install/active_release/bin:$PATH" >> $GITHUB_ENV - export PATH="/home/runner/.local/share/solana/install/active_release/bin:$PATH" - cargo build-bpf --version - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" - sudo apt-get update - sudo apt-get install -y clang-7 --allow-unauthenticated - sudo apt-get install -y openssl --allow-unauthenticated - sudo apt-get install -y libssl-dev --allow-unauthenticated - sudo apt-get install -y libssl1.1 --allow-unauthenticated - sudo apt-get install -y libudev-dev - clang-7 --version - - - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - ${{ matrix.path }}/target - key: cargo-excluded-${{ matrix.path }}-${{ hashFiles('**/Cargo.lock') }}-nonce-1 - restore-keys: | - cargo-excluded-${{ matrix.path }}- - - - uses: actions/download-artifact@v2 - with: - name: programs - path: target/bpfel-unknown-unknown/release - - - name: Run tests - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path=${{ matrix.path }}/Cargo.toml -- --nocapture - - js_token: + js-test-token: runs-on: ubuntu-latest env: NODE_VERSION: 12.x - defaults: - run: - working-directory: token/js - needs: build_bpf_programs + needs: cargo-build-test steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ env.NODE_VERSION }} @@ -303,30 +148,18 @@ jobs: key: node-token-${{ hashFiles('**/package-lock.json') }} restore-keys: | node-token- - - uses: actions/download-artifact@v2 + - name: Download programs + uses: actions/download-artifact@v2 with: name: programs path: target/bpfel-unknown-unknown/release - - run: npm install - - run: npm run lint - - run: npm run flow - - run: tsc module.d.ts - - run: | - npm run cluster:localnet - npm run localnet:update - npm run localnet:up - - run: npm run start - - run: PROGRAM_VERSION=2.0.4 npm run start - - run: npm run localnet:down + - run: ./ci/js-test-token.sh - js_token_swap: + js-test-token-swap: runs-on: ubuntu-latest env: NODE_VERSION: 12.x - defaults: - run: - working-directory: token-swap/js - needs: build_bpf_programs + needs: cargo-build-test steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ env.NODE_VERSION }} @@ -338,35 +171,19 @@ jobs: path: ~/.npm key: node-token-swap-${{ hashFiles('**/package-lock.json') }} restore-keys: | - node-token-swap - - uses: actions/download-artifact@v2 + node-token-swap- + - name: Download programs + uses: actions/download-artifact@v2 with: name: programs path: target/bpfel-unknown-unknown/release - - run: (cd ../../token/js && npm install) - - run: npm install - - run: npm run lint - - run: npm run flow - - run: tsc module.d.ts - - run: | - npm run cluster:localnet - npm run localnet:update - npm run localnet:up - - run: npm run start - - name: run production test - run: | - (cd ../../target/bpfel-unknown-unknown/release && mv spl_token_swap_production.so spl_token_swap.so) - SWAP_PROGRAM_OWNER_FEE_ADDRESS="SwaPpA9LAaLfeLi3a68M4DjnLqgtticKg6CnyNwgAC8" npm run start - - run: npm run localnet:down + - run: ./ci/js-test-token-swap.sh - js_token_lending: + js-test-token-lending: runs-on: ubuntu-latest env: NODE_VERSION: 12.x - defaults: - run: - working-directory: token-lending/js - needs: build_bpf_programs + needs: cargo-build-test steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ env.NODE_VERSION }} @@ -379,17 +196,9 @@ jobs: key: node-token-lending-${{ hashFiles('**/package-lock.json') }} restore-keys: | node-token-lending- - - uses: actions/download-artifact@v2 + - name: Download programs + uses: actions/download-artifact@v2 with: name: programs path: target/bpfel-unknown-unknown/release - - run: (cd ../../token/js && npm install) - - run: npm install - - run: npm run lint - - run: npm run build - - run: | - npm run cluster:localnet - npm run localnet:update - npm run localnet:up - - run: npm run start - - run: npm run localnet:down + - run: ./ci/js-test-token-lending.sh diff --git a/.travis.yml b/.travis.yml index be212de8..162cc8bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,18 +8,6 @@ branches: notifications: email: false -cache: - directories: - - "~/.cache" - -services: - - docker - -env: - global: - - RUST_BACKTRACE=1 - - SOLANA_VERSION=v1.4.3 - jobs: include: # docs pull request or commit to master diff --git a/ci/cargo-build-test.sh b/ci/cargo-build-test.sh new file mode 100755 index 00000000..247f18f9 --- /dev/null +++ b/ci/cargo-build-test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -e +cd "$(dirname "$0")" +cd .. + +source ./ci/rust-version.sh stable +source ./ci/solana-version.sh install + +export RUSTFLAGS="-D warnings" +export RUSTBACKTRACE=1 + +set -x + +# For all BPF programs +for Xargo_toml in $(git ls-files -- '*/Xargo.toml'); do + program_dir=$(dirname "$Xargo_toml") + + if [ "$program_dir" == "token-swap/program" ]; then + address="SwaPpA9LAaLfeLi3a68M4DjnLqgtticKg6CnyNwgAC8" + SWAP_PROGRAM_OWNER_FEE_ADDRESS="$address" cargo build-bpf --manifest-path=token-swap/program/Cargo.toml --dump --features production + mv spl_token_swap.so spl_token_swap_production.so + fi + + cargo +"$rust_stable" build-bpf --manifest-path="$program_dir"/Cargo.toml --dump +done + + +cargo +"$rust_stable" build +cargo +"$rust_stable" test -- --nocapture +cargo +"$rust_stable" run --manifest-path=utils/test-client/Cargo.toml +cargo +"$rust_stable" test --manifest-path=themis/client_ristretto/Cargo.toml -- --nocapture + +# # Check generated C headers +# cargo run --manifest-path=utils/cgen/Cargo.toml +# +# git diff --exit-code token/program/inc/token.h +# cc token/program/inc/token.h -o target/token.gch +# git diff --exit-code token-swap/program/inc/token-swap.h +# cc token-swap/program/inc/token-swap.h -o target/token-swap.gch + +exit 0 diff --git a/ci/install-build-deps.sh b/ci/install-build-deps.sh new file mode 100755 index 00000000..53de34b3 --- /dev/null +++ b/ci/install-build-deps.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex + +wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - +sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" +sudo apt-get update +sudo apt-get install -y clang-7 --allow-unauthenticated +sudo apt-get install -y openssl --allow-unauthenticated +sudo apt-get install -y libssl-dev --allow-unauthenticated +sudo apt-get install -y libssl1.1 --allow-unauthenticated +sudo apt-get install -y libudev-dev +clang-7 --version diff --git a/ci/install-program-deps.sh b/ci/install-program-deps.sh new file mode 100755 index 00000000..d2aa9ea3 --- /dev/null +++ b/ci/install-program-deps.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e + +source ci/rust-version.sh stable +source ci/solana-version.sh install + +set -x + +cargo --version +cargo install rustfilt || true + +export PATH="$HOME"/.local/share/solana/install/active_release/bin:"$PATH" +solana --version +cargo +"$rust_stable" build-bpf --version diff --git a/ci/js-test-token-lending.sh b/ci/js-test-token-lending.sh new file mode 100755 index 00000000..531a80ae --- /dev/null +++ b/ci/js-test-token-lending.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "$0")" + +(cd ../token/js && npm install) + +cd ../token-lending/js +npm install +npm run lint +npm run build +npm run cluster:localnet +npm run localnet:update +npm run localnet:up +npm run start +npm run localnet:down \ No newline at end of file diff --git a/ci/js-test-token-swap.sh b/ci/js-test-token-swap.sh new file mode 100755 index 00000000..16335279 --- /dev/null +++ b/ci/js-test-token-swap.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "$0")" + +(cd ../token/js && npm install) + +cd ../token-swap/js +npm install +npm run lint +npm run flow +npx tsc module.d.ts +npm run cluster:localnet +npm run localnet:update +npm run localnet:up +npm run start +(cd ../../target/bpfel-unknown-unknown/release && mv spl_token_swap_production.so spl_token_swap.so) +SWAP_PROGRAM_OWNER_FEE_ADDRESS="SwaPpA9LAaLfeLi3a68M4DjnLqgtticKg6CnyNwgAC8" npm run start +npm run localnet:down \ No newline at end of file diff --git a/ci/js-test-token.sh b/ci/js-test-token.sh new file mode 100755 index 00000000..95f0941f --- /dev/null +++ b/ci/js-test-token.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "$0")" +cd ../token/js + +npm install +npm run lint +npm run flow +npx tsc module.d.ts +npm run cluster:localnet +npm run localnet:update +npm run localnet:up +npm run start +PROGRAM_VERSION=2.0.4 npm run start +npm run localnet:down \ No newline at end of file diff --git a/ci/rust-version.sh b/ci/rust-version.sh new file mode 100644 index 00000000..874a2dd4 --- /dev/null +++ b/ci/rust-version.sh @@ -0,0 +1,65 @@ +# +# This file maintains the rust versions for use by CI. +# +# Obtain the environment variables without any automatic toolchain updating: +# $ source ci/rust-version.sh +# +# Obtain the environment variables updating both stable and nightly, only stable, or +# only nightly: +# $ source ci/rust-version.sh all +# $ source ci/rust-version.sh stable +# $ source ci/rust-version.sh nightly + +# Then to build with either stable or nightly: +# $ cargo +"$rust_stable" build +# $ cargo +"$rust_nightly" build +# + +if [[ -n $RUST_STABLE_VERSION ]]; then + stable_version="$RUST_STABLE_VERSION" +else + stable_version=1.47.0 +fi + +# if [[ -n $RUST_NIGHTLY_VERSION ]]; then +# nightly_version="$RUST_NIGHTLY_VERSION" +# else +# nightly_version=2020-08-17 +# fi + + +export rust_stable="$stable_version" +export rust_stable_docker_image=solanalabs/rust:"$stable_version" + +# export rust_nightly=nightly-"$nightly_version" +# export rust_nightly_docker_image=solanalabs/rust-nightly:"$nightly_version" + +[[ -z $1 ]] || ( + + rustup_install() { + declare toolchain=$1 + if ! cargo +"$toolchain" -V > /dev/null; then + echo "$0: Missing toolchain? Installing...: $toolchain" >&2 + rustup install "$toolchain" + cargo +"$toolchain" -V + fi + } + + set -e + cd "$(dirname "${BASH_SOURCE[0]}")" + case $1 in + stable) + rustup_install "$rust_stable" + ;; + # nightly) + # rustup_install "$rust_nightly" + # ;; + all) + rustup_install "$rust_stable" + rustup_install "$rust_nightly" + ;; + *) + echo "$0: Note: ignoring unknown argument: $1" >&2 + ;; + esac +) diff --git a/ci/solana-version.sh b/ci/solana-version.sh new file mode 100755 index 00000000..774f4138 --- /dev/null +++ b/ci/solana-version.sh @@ -0,0 +1,35 @@ +# +# This file maintains the solana versions for use by CI. +# +# Obtain the environment variables without any automatic updating: +# $ source ci/solana-version.sh +# +# Obtain the environment variables and install update: +# $ source ci/solana-version.sh install + +# Then to access the solana version: +# $ echo "$solana_version" +# + +if [[ -n $SOLANA_VERSION ]]; then + solana_version="$SOLANA_VERSION" +else + solana_version=v1.4.4 +fi + +export solana_version="$solana_version" +export solana_docker_image=solanalabs/solana:"$solana_version" + +[[ -z $1 ]] || ( + + set -e + cd "$(dirname "${BASH_SOURCE[0]}")" + case $1 in + install) + sh -c "$(curl -sSfL https://release.solana.com/$solana_version/install)" + ;; + *) + echo "$0: Note: ignoring unknown argument: $1" >&2 + ;; + esac +)