diff --git a/.buildkite/pipeline-upload.sh b/.buildkite/pipeline-upload.sh index d256c2767..dd3aa146c 100755 --- a/.buildkite/pipeline-upload.sh +++ b/.buildkite/pipeline-upload.sh @@ -14,7 +14,7 @@ buildkite-agent pipeline upload ci/buildkite.yml if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request - buildkite-agent annotate --style "info" \ + buildkite-agent annotate --style info --context pr-backlink \ "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" fi diff --git a/README.md b/README.md index f43fb3424..2a8774f91 100644 --- a/README.md +++ b/README.md @@ -150,22 +150,13 @@ The release process for this project is described [here](RELEASE.md). Code coverage --- -To generate code coverage statistics, install cargo-cov. Note: the tool currently only works -in Rust nightly. +To generate code coverage statistics: ```bash -$ cargo +nightly install cargo-cov +$ scripts/coverage.sh +$ open target/cov/lcov-local/index.html ``` -Run cargo-cov and generate a report: - -```bash -$ cargo +nightly cov test -$ cargo +nightly cov report --open -``` - -The coverage report will be written to `./target/cov/report/index.html` - Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to diff --git a/ci/buildkite.yml b/ci/buildkite.yml index 52e2ca348..9f4991684 100644 --- a/ci/buildkite.yml +++ b/ci/buildkite.yml @@ -17,9 +17,9 @@ steps: - command: "ci/docker-run.sh solanalabs/rust:1.31.0 ci/test-stable.sh" name: "stable" timeout_in_minutes: 30 - #- command: "ci/docker-run.sh solanalabs/rust-nightly:2018-12-05 ci/test-nightly.sh" - # name: "nightly" - # timeout_in_minutes: 30 + - command: "ci/docker-run.sh solanalabs/rust-nightly:2018-12-18 ci/test-coverage.sh" + name: "coverage" + timeout_in_minutes: 30 # TODO: Fix and re-enable test-large-network.sh # - command: "ci/test-large-network.sh || true" # name: "large-network [ignored]" diff --git a/ci/test-coverage.sh b/ci/test-coverage.sh new file mode 100755 index 000000000..14278bce6 --- /dev/null +++ b/ci/test-coverage.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -e + +cd "$(dirname "$0")/.." + +annotate() { + ${BUILDKITE:-false} && { + buildkite-agent annotate "$@" + } +} + +affectedFiles="$(buildkite-agent meta-data get affected_files)" +echo "Affected files in this PR: $affectedFiles" +if [[ ! ":$affectedFiles:" =~ \.rs: ]]; then + annotate --style info --context coverage-info \ + "Coverage skipped as no .rs files were modified" + exit 0 +fi + +source ci/upload-ci-artifact.sh +ci/version-check-with-upgrade.sh nightly + +scripts/coverage.sh + +report=coverage-"${BUILDKITE_COMMIT:0:9}".tar.gz +mv target/cov/report.tar.gz "$report" +upload-ci-artifact "$report" +annotate --style success --context lcov-report \ + "lcov report: $report" + +echo "--- codecov.io report" +if [[ -z "$CODECOV_TOKEN" ]]; then + echo "^^^ +++" + echo CODECOV_TOKEN undefined, codecov.io upload skipped +else + bash <(curl -s https://codecov.io/bash) -X gcov -f target/cov/lcov.info + + annotate --style success --context codecov.io \ + "CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${BUILDKITE_COMMIT:0:9}" +fi diff --git a/ci/test-nightly.sh b/ci/test-nightly.sh deleted file mode 100755 index 329002f6d..000000000 --- a/ci/test-nightly.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -e - -cd "$(dirname "$0")/.." -source ci/upload-ci-artifact.sh - -ci/version-check.sh nightly -export RUST_BACKTRACE=1 - -_() { - echo "--- $*" - "$@" -} - -# Uncomment this to run nightly test suit -# _ cargo test --all --verbose --features=unstable -- --test-threads=1 - -cargo_install_unless() { - declare crate=$1 - shift - - "$@" > /dev/null 2>&1 || \ - _ cargo install "$crate" -} - -cargo_install_unless cargo-cov cargo cov --help - -# Generate coverage data and report via unit-test suite. -_ cargo cov clean -_ cargo cov build --all -_ cargo cov test --lib -_ cargo cov report -_ ./scripts/fetch-grcov.sh -_ ./grcov . -t lcov > lcov.info -_ genhtml -o target/cov/report-lcov --show-details --highlight --ignore-errors source --legend lcov.info - -# Upload to tarballs to buildkite. -_ cd target/cov && tar -cjf cov-report.tar.bz2 report/* && cd - -_ upload-ci-artifact "target/cov/cov-report.tar.bz2" - -_ cd target/cov && tar -cjf lcov-report.tar.bz2 report-lcov/* && cd - -_ upload-ci-artifact "target/cov/lcov-report.tar.bz2" - -# Upload coverage files to buildkite for grcov debugging -_ cd target/cov/build && tar -cjf cov-gcda.tar.bz2 gcda/* && cd - -_ upload-ci-artifact "target/cov/build/cov-gcda.tar.bz2" - -_ cd target/cov/build && tar -cjf cov-gcno.tar.bz2 gcno/* && cd - -_ upload-ci-artifact "target/cov/build/cov-gcno.tar.bz2" - -if [[ -z "$CODECOV_TOKEN" ]]; then - echo CODECOV_TOKEN undefined -else - true - # TODO: Why doesn't codecov grok our lcov files? - #bash <(curl -s https://codecov.io/bash) -X gcov -fi diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100755 index 000000000..2e2d05392 --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# +# Runs all tests and collects code coverage +# +# Warning: this process is a little slow +# + +set -e +cd "$(dirname "$0")/.." + +_() { + echo "--- $*" + "$@" +} + +: "${BUILDKITE_COMMIT:=local}" +reportName="lcov-${BUILDKITE_COMMIT:0:9}" + +export RUSTFLAGS=" + -Zprofile + -Zno-landing-pads + -Ccodegen-units=1 + -Cinline-threshold=0 + -Coverflow-checks=off +" +export CARGO_INCREMENTAL=0 +export RUST_BACKTRACE=1 + +echo "--- remove old coverage results" + +if [[ -d target/cov ]]; then + find target/cov -name \*.gcda -print0 | xargs -0 rm -f +fi +rm -rf target/cov/$reportName + +_ cargo +nightly build --target-dir target/cov --all +_ cargo +nightly test --target-dir target/cov --lib --all + +_ scripts/fetch-grcov.sh +echo "--- grcov" +./grcov target/cov/debug/deps/ > target/cov/lcov_full.info + +echo "--- filter_non_local_files_from_lcov" +filter_non_local_files_from_lcov() { + declare skip=false + while read -r line; do + if [[ $line =~ ^SF:/ ]]; then + skip=true # Skip all absolute paths as these are references into ~/.cargo + elif [[ $line =~ ^SF:(.*) ]]; then + # Skip relative paths that don't exist + declare file="${BASH_REMATCH[1]}" + if [[ -r $file ]]; then + skip=false + else + skip=true + fi + fi + [[ $skip = true ]] || echo "$line" + done +} + +filter_non_local_files_from_lcov < target/cov/lcov_full.info > target/cov/lcov.info + +echo "--- html report" +# ProTip: genhtml comes from |brew install lcov| or |apt-get install lcov| +genhtml --output-directory target/cov/$reportName \ + --show-details \ + --highlight \ + --ignore-errors source \ + --prefix "$PWD" \ + --legend \ + target/cov/lcov.info + +( + cd target/cov + tar zcf report.tar.gz $reportName +) + +ls -l target/cov/$reportName/index.html