switch to source based coverage (#1293)

* switch to new llvm source based coverage

* upload artifact and simplified

* filter out irrelevant dependency coverage

* enable the correct filters on coverage

* correctly specify all binaries

* remove sparse flag from coverage

* update the coverage script organization

* fix typo in coverage script

.github/workflows/coverage.yml

@@ -13,20 +13,46 @@ jobs:
   coverage:
     name: Coverage
     timeout-minutes: 30
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
+
       - uses: actions-rs/toolchain@v1
         with:
-          toolchain: stable
+          toolchain: nightly
           override: true
-      - name: Skip network tests on Ubuntu
+          profile: minimal
-        # Ubuntu runners don't have network or DNS configured during test steps
+          components: llvm-tools-preview
-        run: echo "ZEBRA_SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV
+
-      - name: Run cargo-tarpaulin
+      - name: Install rustfilt symbol demangler
-        uses: actions-rs/tarpaulin@v0.1
+        run: |
+          cargo install rustfilt
+
+      - name: Rerun tests for coverage
+        env:
+          RUSTFLAGS: -Zinstrument-coverage -C link-dead-code -C debuginfo=2
+          LLVM_PROFILE_FILE: "${{ github.workspace }}/test.%p.profraw"
+          ZEBRA_SKIP_NETWORK_TESTS: 1
+        run: |
+          cargo test
+          cargo test --no-run --message-format=json | jq -r "select(.profile.test == true) | .filenames[]" | grep -v dSYM - > filenames.txt
+
+      - name: Merge coverage data
+        run: |
+          $(rustc --print target-libdir)/../bin/llvm-profdata merge test.*.profraw -o test.profdata
+
+      - name: Generate detailed html coverage report for github artifact
+        run: |
+          $(rustc --print target-libdir)/../bin/llvm-cov show -format=html -ignore-filename-regex=".*/.cargo/registry/.*" -ignore-filename-regex=".*/.cargo/git/.*" -ignore-filename-regex=".*/.rustup/.*" -Xdemangler=rustfilt -show-instantiations -output-dir=./coverage -instr-profile=./test.profdata $(printf -- "-object %s " $(cat filenames.txt))
+
+      - uses: actions/upload-artifact@v2
         with:
-          version: '0.16.0'
+          name: coverage
-          timeout: 600
+          path: ./coverage
+
+      - name: Generate lcov coverage report for codecov
+        run: |
+          $(rustc --print target-libdir)/../bin/llvm-cov export -format=lcov -instr-profile=test.profdata $(printf -- "-object %s " $(cat filenames.txt)) > "lcov.info"
+
       - name: Upload coverage report to Codecov
-        uses: codecov/codecov-action@v1.0.15
+        uses: codecov/codecov-action@v1

.gitignore

@@ -1,5 +1,6 @@
 # Cargo files
 /target/
+/coverage-target/
 # Firebase caches (?)
 .firebase/
 # Emacs detritus

zebra-utils/coverage

@@ -0,0 +1,34 @@
+#! /bin/bash
+set -e
+set -o xtrace
+
+export CARGO_TARGET_DIR="coverage-target"
+export ZEBRA_SKIP_NETWORK_TESTS=1
+export LLVM_PROFILE_FILE="${PWD}/$CARGO_TARGET_DIR/coverage/test.%m.profraw"
+export RUSTFLAGS="-Zinstrument-coverage -C link-dead-code -C debuginfo=2"
+
+rm -rf ./"$CARGO_TARGET_DIR"/coverage
+mkdir -p ./$CARGO_TARGET_DIR/coverage
+cargo +nightly test
+cargo +nightly test --no-run --message-format=json | jq -r "select(.profile.test == true) | .filenames[]" | grep -v dSYM - > ./$CARGO_TARGET_DIR/files.txt
+$(rustc +nightly --print target-libdir)/../bin/llvm-profdata merge ./$CARGO_TARGET_DIR/coverage/test.*.profraw -o ./$CARGO_TARGET_DIR/coverage/test.profdata
+
+rm -rf ./$CARGO_TARGET_DIR/coverage/html/
+
+$(rustc +nightly --print target-libdir)/../bin/llvm-cov show \
+	-format=html \
+	-Xdemangler=rustfilt \
+	-show-instantiations \
+	-output-dir=./$CARGO_TARGET_DIR/coverage/html \
+	-ignore-filename-regex=".*/.cargo/registry/.*" \
+	-ignore-filename-regex=".*/.cargo/git/.*" \
+	-ignore-filename-regex=".*/.rustup/.*" \
+	-instr-profile=./$CARGO_TARGET_DIR/coverage/test.profdata \
+	$(printf -- "-object %s " $(cat ./$CARGO_TARGET_DIR/files.txt))
+
+$(rustc +nightly --print target-libdir)/../bin/llvm-cov export \
+	-format=lcov \
+	-instr-profile=./$CARGO_TARGET_DIR/coverage/test.profdata \
+	$(printf -- "-object %s " $(cat ./$CARGO_TARGET_DIR/files.txt)) > "./$CARGO_TARGET_DIR/coverage/lcov.info"
+
+xdg-open ./$CARGO_TARGET_DIR/coverage/html/index.html