Overhaul coverage setup
This commit is contained in:
Michael Vines
parent
c71d5a111e
commit
3bfb052b0a
|
|
@ -14,7 +14,7 @@ buildkite-agent pipeline upload ci/buildkite.yml
|
|||
|
||||
if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
|
||||
# Add helpful link back to the corresponding Github Pull Request
|
||||
buildkite-agent annotate --style "info" \
|
||||
buildkite-agent annotate --style info --context pr-backlink \
|
||||
"Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
|
||||
fi
|
||||
|
||||
|
|
|
|||
15
README.md
15
README.md
|
|
@ -150,22 +150,13 @@ The release process for this project is described [here](RELEASE.md).
|
|||
Code coverage
|
||||
---
|
||||
|
||||
To generate code coverage statistics, install cargo-cov. Note: the tool currently only works
|
||||
in Rust nightly.
|
||||
To generate code coverage statistics:
|
||||
|
||||
```bash
|
||||
$ cargo +nightly install cargo-cov
|
||||
$ scripts/coverage.sh
|
||||
$ open target/cov/lcov-local/index.html
|
||||
```
|
||||
|
||||
Run cargo-cov and generate a report:
|
||||
|
||||
```bash
|
||||
$ cargo +nightly cov test
|
||||
$ cargo +nightly cov report --open
|
||||
```
|
||||
|
||||
The coverage report will be written to `./target/cov/report/index.html`
|
||||
|
||||
|
||||
Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer
|
||||
productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@ steps:
|
|||
- command: "ci/docker-run.sh solanalabs/rust:1.31.0 ci/test-stable.sh"
|
||||
name: "stable"
|
||||
timeout_in_minutes: 30
|
||||
#- command: "ci/docker-run.sh solanalabs/rust-nightly:2018-12-05 ci/test-nightly.sh"
|
||||
# name: "nightly"
|
||||
# timeout_in_minutes: 30
|
||||
- command: "ci/docker-run.sh solanalabs/rust-nightly:2018-12-18 ci/test-coverage.sh"
|
||||
name: "coverage"
|
||||
timeout_in_minutes: 30
|
||||
# TODO: Fix and re-enable test-large-network.sh
|
||||
# - command: "ci/test-large-network.sh || true"
|
||||
# name: "large-network [ignored]"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
annotate() {
|
||||
${BUILDKITE:-false} && {
|
||||
buildkite-agent annotate "$@"
|
||||
}
|
||||
}
|
||||
|
||||
affectedFiles="$(buildkite-agent meta-data get affected_files)"
|
||||
echo "Affected files in this PR: $affectedFiles"
|
||||
if [[ ! ":$affectedFiles:" =~ \.rs: ]]; then
|
||||
annotate --style info --context coverage-info \
|
||||
"Coverage skipped as no .rs files were modified"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
source ci/upload-ci-artifact.sh
|
||||
ci/version-check-with-upgrade.sh nightly
|
||||
|
||||
scripts/coverage.sh
|
||||
|
||||
report=coverage-"${BUILDKITE_COMMIT:0:9}".tar.gz
|
||||
mv target/cov/report.tar.gz "$report"
|
||||
upload-ci-artifact "$report"
|
||||
annotate --style success --context lcov-report \
|
||||
"lcov report: <a href=\"artifact://$report\">$report</a>"
|
||||
|
||||
echo "--- codecov.io report"
|
||||
if [[ -z "$CODECOV_TOKEN" ]]; then
|
||||
echo "^^^ +++"
|
||||
echo CODECOV_TOKEN undefined, codecov.io upload skipped
|
||||
else
|
||||
bash <(curl -s https://codecov.io/bash) -X gcov -f target/cov/lcov.info
|
||||
|
||||
annotate --style success --context codecov.io \
|
||||
"CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${BUILDKITE_COMMIT:0:9}"
|
||||
fi
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
source ci/upload-ci-artifact.sh
|
||||
|
||||
ci/version-check.sh nightly
|
||||
export RUST_BACKTRACE=1
|
||||
|
||||
_() {
|
||||
echo "--- $*"
|
||||
"$@"
|
||||
}
|
||||
|
||||
# Uncomment this to run nightly test suit
|
||||
# _ cargo test --all --verbose --features=unstable -- --test-threads=1
|
||||
|
||||
cargo_install_unless() {
|
||||
declare crate=$1
|
||||
shift
|
||||
|
||||
"$@" > /dev/null 2>&1 || \
|
||||
_ cargo install "$crate"
|
||||
}
|
||||
|
||||
cargo_install_unless cargo-cov cargo cov --help
|
||||
|
||||
# Generate coverage data and report via unit-test suite.
|
||||
_ cargo cov clean
|
||||
_ cargo cov build --all
|
||||
_ cargo cov test --lib
|
||||
_ cargo cov report
|
||||
_ ./scripts/fetch-grcov.sh
|
||||
_ ./grcov . -t lcov > lcov.info
|
||||
_ genhtml -o target/cov/report-lcov --show-details --highlight --ignore-errors source --legend lcov.info
|
||||
|
||||
# Upload to tarballs to buildkite.
|
||||
_ cd target/cov && tar -cjf cov-report.tar.bz2 report/* && cd -
|
||||
_ upload-ci-artifact "target/cov/cov-report.tar.bz2"
|
||||
|
||||
_ cd target/cov && tar -cjf lcov-report.tar.bz2 report-lcov/* && cd -
|
||||
_ upload-ci-artifact "target/cov/lcov-report.tar.bz2"
|
||||
|
||||
# Upload coverage files to buildkite for grcov debugging
|
||||
_ cd target/cov/build && tar -cjf cov-gcda.tar.bz2 gcda/* && cd -
|
||||
_ upload-ci-artifact "target/cov/build/cov-gcda.tar.bz2"
|
||||
|
||||
_ cd target/cov/build && tar -cjf cov-gcno.tar.bz2 gcno/* && cd -
|
||||
_ upload-ci-artifact "target/cov/build/cov-gcno.tar.bz2"
|
||||
|
||||
if [[ -z "$CODECOV_TOKEN" ]]; then
|
||||
echo CODECOV_TOKEN undefined
|
||||
else
|
||||
true
|
||||
# TODO: Why doesn't codecov grok our lcov files?
|
||||
#bash <(curl -s https://codecov.io/bash) -X gcov
|
||||
fi
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Runs all tests and collects code coverage
|
||||
#
|
||||
# Warning: this process is a little slow
|
||||
#
|
||||
|
||||
set -e
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
_() {
|
||||
echo "--- $*"
|
||||
"$@"
|
||||
}
|
||||
|
||||
: "${BUILDKITE_COMMIT:=local}"
|
||||
reportName="lcov-${BUILDKITE_COMMIT:0:9}"
|
||||
|
||||
export RUSTFLAGS="
|
||||
-Zprofile
|
||||
-Zno-landing-pads
|
||||
-Ccodegen-units=1
|
||||
-Cinline-threshold=0
|
||||
-Coverflow-checks=off
|
||||
"
|
||||
export CARGO_INCREMENTAL=0
|
||||
export RUST_BACKTRACE=1
|
||||
|
||||
echo "--- remove old coverage results"
|
||||
|
||||
if [[ -d target/cov ]]; then
|
||||
find target/cov -name \*.gcda -print0 | xargs -0 rm -f
|
||||
fi
|
||||
rm -rf target/cov/$reportName
|
||||
|
||||
_ cargo +nightly build --target-dir target/cov --all
|
||||
_ cargo +nightly test --target-dir target/cov --lib --all
|
||||
|
||||
_ scripts/fetch-grcov.sh
|
||||
echo "--- grcov"
|
||||
./grcov target/cov/debug/deps/ > target/cov/lcov_full.info
|
||||
|
||||
echo "--- filter_non_local_files_from_lcov"
|
||||
filter_non_local_files_from_lcov() {
|
||||
declare skip=false
|
||||
while read -r line; do
|
||||
if [[ $line =~ ^SF:/ ]]; then
|
||||
skip=true # Skip all absolute paths as these are references into ~/.cargo
|
||||
elif [[ $line =~ ^SF:(.*) ]]; then
|
||||
# Skip relative paths that don't exist
|
||||
declare file="${BASH_REMATCH[1]}"
|
||||
if [[ -r $file ]]; then
|
||||
skip=false
|
||||
else
|
||||
skip=true
|
||||
fi
|
||||
fi
|
||||
[[ $skip = true ]] || echo "$line"
|
||||
done
|
||||
}
|
||||
|
||||
filter_non_local_files_from_lcov < target/cov/lcov_full.info > target/cov/lcov.info
|
||||
|
||||
echo "--- html report"
|
||||
# ProTip: genhtml comes from |brew install lcov| or |apt-get install lcov|
|
||||
genhtml --output-directory target/cov/$reportName \
|
||||
--show-details \
|
||||
--highlight \
|
||||
--ignore-errors source \
|
||||
--prefix "$PWD" \
|
||||
--legend \
|
||||
target/cov/lcov.info
|
||||
|
||||
(
|
||||
cd target/cov
|
||||
tar zcf report.tar.gz $reportName
|
||||
)
|
||||
|
||||
ls -l target/cov/$reportName/index.html
|
||||
Loading…
Reference in New Issue