From 3501399f286b7820bffd409036eefd9048a6f2bb Mon Sep 17 00:00:00 2001 From: Jane Lusby Date: Fri, 27 Nov 2020 14:29:59 -0800 Subject: [PATCH 1/8] switch to new llvm source based coverage --- .github/workflows/coverage.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 10875a7e46a..f85c11503af 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,20 +13,26 @@ jobs: coverage: name: Coverage timeout-minutes: 30 - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: nightly override: true - - name: Skip network tests on Ubuntu - # Ubuntu runners don't have network or DNS configured during test steps - run: echo "ZEBRA_SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV - - name: Run cargo-tarpaulin - uses: actions-rs/tarpaulin@v0.1 - with: - version: '0.16.0' - timeout: 600 + profile: minimal + components: llvm-tools-preview + - name: Install rustfilt symbol demangler + run: cargo install rustfilt + - name: Rerun tests for coverage + run: cargo test + env: + RUSTFLAGS: -Zinstrument-coverage + LLVM_PROFILE_FILE: "${{ github.workspace }}/test.%p.profraw" + ZEBRA_SKIP_NETWORK_TESTS: 1 + - name: Merge coverage data + run: $(rustc --print target-libdir)/../bin/llvm-profdata merge --sparse test.*.profraw -o test.profdata + - name: Generate coverage report + run: $(rustc --print target-libdir)/../bin/llvm-cov export -format=lcov -instr-profile=test.profdata $(find target/debug/deps -type f -perm -u+x ! -name '*.so') > "lcov.info" - name: Upload coverage report to Codecov - uses: codecov/codecov-action@v1.0.15 + uses: codecov/codecov-action@v1 From 7dac8f2a3fb32b41aa7002e97fa8af6a6916709f Mon Sep 17 00:00:00 2001 From: Jane Lusby Date: Fri, 27 Nov 2020 17:55:37 -0800 Subject: [PATCH 2/8] upload artifact and simplified --- .github/workflows/coverage.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f85c11503af..0fb7900bf97 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -32,7 +32,13 @@ jobs: ZEBRA_SKIP_NETWORK_TESTS: 1 - name: Merge coverage data run: $(rustc --print target-libdir)/../bin/llvm-profdata merge --sparse test.*.profraw -o test.profdata - - name: Generate coverage report + - name: Generate detailed html coverage report for github artifact + run: $(rustc --print target-libdir)/../bin/llvm-cov show -format=html -Xdemangler=rustfilt -show-instantiations -output-dir=./coverage -instr-profile=./test.profdata $(find target/debug/deps -type f -perm -u+x ! -name '*.so') + - uses: actions/upload-artifact@v2 + with: + name: coverage + path: ./coverage + - name: Generate lcov coverage report for codecov run: $(rustc --print target-libdir)/../bin/llvm-cov export -format=lcov -instr-profile=test.profdata $(find target/debug/deps -type f -perm -u+x ! -name '*.so') > "lcov.info" - name: Upload coverage report to Codecov uses: codecov/codecov-action@v1 From ec08d6271f1d5c679beef763c6f9dfba90d3129b Mon Sep 17 00:00:00 2001 From: Jane Lusby Date: Fri, 27 Nov 2020 17:59:05 -0800 Subject: [PATCH 3/8] filter out irrelevant dependency coverage --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 0fb7900bf97..db935ce005b 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -33,7 +33,7 @@ jobs: - name: Merge coverage data run: $(rustc --print target-libdir)/../bin/llvm-profdata merge --sparse test.*.profraw -o test.profdata - name: Generate detailed html coverage report for github artifact - run: $(rustc --print target-libdir)/../bin/llvm-cov show -format=html -Xdemangler=rustfilt -show-instantiations -output-dir=./coverage -instr-profile=./test.profdata $(find target/debug/deps -type f -perm -u+x ! -name '*.so') + run: $(rustc --print target-libdir)/../bin/llvm-cov show -format=html -ignore-filename-regex="*/.cargo/registry/*" -Xdemangler=rustfilt -show-instantiations -output-dir=./coverage -instr-profile=./test.profdata $(find target/debug/deps -type f -perm -u+x ! -name '*.so') - uses: actions/upload-artifact@v2 with: name: coverage From a9303375ba0c24afea0c65dcdbf8184d236f42dc Mon Sep 17 00:00:00 2001 From: Jane Lusby Date: Sat, 28 Nov 2020 09:43:30 -0800 Subject: [PATCH 4/8] enable the correct filters on coverage --- .github/workflows/coverage.yml | 2 +- zebra-utils/coverage | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100755 zebra-utils/coverage diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index db935ce005b..a889c0c01a7 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -33,7 +33,7 @@ jobs: - name: Merge coverage data run: $(rustc --print target-libdir)/../bin/llvm-profdata merge --sparse test.*.profraw -o test.profdata - name: Generate detailed html coverage report for github artifact - run: $(rustc --print target-libdir)/../bin/llvm-cov show -format=html -ignore-filename-regex="*/.cargo/registry/*" -Xdemangler=rustfilt -show-instantiations -output-dir=./coverage -instr-profile=./test.profdata $(find target/debug/deps -type f -perm -u+x ! -name '*.so') + run: $(rustc --print target-libdir)/../bin/llvm-cov show -format=html -ignore-filename-regex=".*/.cargo/registry/.*" -ignore-filename-regex=".*/.cargo/git/.*" -ignore-filename-regex=".*/.rustup/.*" -Xdemangler=rustfilt -show-instantiations -output-dir=./coverage -instr-profile=./test.profdata $(find target/debug/deps -type f -perm -u+x ! -name '*.so') - uses: actions/upload-artifact@v2 with: name: coverage diff --git a/zebra-utils/coverage b/zebra-utils/coverage new file mode 100755 index 00000000000..118c428a9a3 --- /dev/null +++ b/zebra-utils/coverage @@ -0,0 +1,23 @@ +#! /bin/bash +set -e +set -o xtrace + +rm -rf ./target/ +mkdir -p ./target/coverage +ZEBRA_SKIP_NETWORK_TESTS=1 LLVM_PROFILE_FILE="${PWD}/target/coverage/test.%p.profraw" RUSTFLAGS="-Zinstrument-coverage" cargo test +$(rustc --print target-libdir)/../bin/llvm-profdata merge --sparse ./target/coverage/test.*.profraw -o ./target/coverage/test.profdata + +rm -rf ./target/coverage/html/ +# This one works and shows all the details I want in the CLI +$(rustc --print target-libdir)/../bin/llvm-cov show \ + -format=html \ + -Xdemangler=rustfilt \ + -show-instantiations \ + -output-dir=./target/coverage/html \ + -ignore-filename-regex=".*/.cargo/registry/.*" \ + -ignore-filename-regex=".*/.cargo/git/.*" \ + -ignore-filename-regex=".*/.rustup/.*" \ + -instr-profile=./target/coverage/test.profdata \ + $(find target/ -type f -perm -u+x ! -name '*.so') + +xdg-open ./target/coverage/html/index.html From f4b1cd2bd0ec8958fc1e229fe5431381a1fc5852 Mon Sep 17 00:00:00 2001 From: Jane Lusby Date: Tue, 1 Dec 2020 14:31:58 -0800 Subject: [PATCH 5/8] correctly specify all binaries --- .github/workflows/coverage.yml | 26 ++++++++++++++++++++------ .gitignore | 1 + zebra-utils/coverage | 28 +++++++++++++++++----------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a889c0c01a7..892f79d0648 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -16,29 +16,43 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 with: toolchain: nightly override: true profile: minimal components: llvm-tools-preview + - name: Install rustfilt symbol demangler - run: cargo install rustfilt + run: | + cargo install rustfilt + - name: Rerun tests for coverage - run: cargo test env: - RUSTFLAGS: -Zinstrument-coverage + RUSTFLAGS: -Zinstrument-coverage -C link-dead-code -C debuginfo=2 LLVM_PROFILE_FILE: "${{ github.workspace }}/test.%p.profraw" ZEBRA_SKIP_NETWORK_TESTS: 1 + run: | + cargo test + cargo test --no-run --message-format=json | jq -r "select(.profile.test == true) | .filenames[]" | grep -v dSYM - > filenames.txt + - name: Merge coverage data - run: $(rustc --print target-libdir)/../bin/llvm-profdata merge --sparse test.*.profraw -o test.profdata + run: | + $(rustc --print target-libdir)/../bin/llvm-profdata merge test.*.profraw -o test.profdata + - name: Generate detailed html coverage report for github artifact - run: $(rustc --print target-libdir)/../bin/llvm-cov show -format=html -ignore-filename-regex=".*/.cargo/registry/.*" -ignore-filename-regex=".*/.cargo/git/.*" -ignore-filename-regex=".*/.rustup/.*" -Xdemangler=rustfilt -show-instantiations -output-dir=./coverage -instr-profile=./test.profdata $(find target/debug/deps -type f -perm -u+x ! -name '*.so') + run: | + $(rustc --print target-libdir)/../bin/llvm-cov show -format=html -ignore-filename-regex=".*/.cargo/registry/.*" -ignore-filename-regex=".*/.cargo/git/.*" -ignore-filename-regex=".*/.rustup/.*" -Xdemangler=rustfilt -show-instantiations -output-dir=./coverage -instr-profile=./test.profdata $(printf -- "-object %s " $(cat filenames.txt)) + - uses: actions/upload-artifact@v2 with: name: coverage path: ./coverage + - name: Generate lcov coverage report for codecov - run: $(rustc --print target-libdir)/../bin/llvm-cov export -format=lcov -instr-profile=test.profdata $(find target/debug/deps -type f -perm -u+x ! -name '*.so') > "lcov.info" + run: | + $(rustc --print target-libdir)/../bin/llvm-cov export -format=lcov -instr-profile=test.profdata $(printf -- "-object %s " $(cat filenames.txt)) > "lcov.info" + - name: Upload coverage report to Codecov uses: codecov/codecov-action@v1 diff --git a/.gitignore b/.gitignore index 8be07b59687..62d81cc249b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Cargo files /target/ +/coverage-target/ # Firebase caches (?) .firebase/ # Emacs detritus diff --git a/zebra-utils/coverage b/zebra-utils/coverage index 118c428a9a3..4c1b5955700 100755 --- a/zebra-utils/coverage +++ b/zebra-utils/coverage @@ -2,22 +2,28 @@ set -e set -o xtrace -rm -rf ./target/ -mkdir -p ./target/coverage -ZEBRA_SKIP_NETWORK_TESTS=1 LLVM_PROFILE_FILE="${PWD}/target/coverage/test.%p.profraw" RUSTFLAGS="-Zinstrument-coverage" cargo test -$(rustc --print target-libdir)/../bin/llvm-profdata merge --sparse ./target/coverage/test.*.profraw -o ./target/coverage/test.profdata +export CARGO_TARGET_DIR="coverage-target" -rm -rf ./target/coverage/html/ -# This one works and shows all the details I want in the CLI -$(rustc --print target-libdir)/../bin/llvm-cov show \ +rm -rf ./"$CARGO_TARGET_DIR"/coverage +mkdir -p ./$CARGO_TARGET_DIR/coverage +export ZEBRA_SKIP_NETWORK_TESTS=1 +export LLVM_PROFILE_FILE="${PWD}/$CARGO_TARGET_DIR/coverage/test.%m.profraw" +export RUSTFLAGS="-Zinstrument-coverage -C link-dead-code -C debuginfo=2" +cargo +nightly test --no-run --message-format=json | jq -r "select(.profile.test == true) | .filenames[]" | grep -v dSYM - > ./$CARGO_TARGET_DIR/files.txt +cargo +nightly test +$(rustc +nightly --print target-libdir)/../bin/llvm-profdata merge --sparse ./$CARGO_TARGET_DIR/coverage/test.*.profraw -o ./$CARGO_TARGET_DIR/coverage/test.profdata + +rm -rf ./$CARGO_TARGET_DIR/coverage/html/ + +$(rustc +nightly --print target-libdir)/../bin/llvm-cov show \ -format=html \ -Xdemangler=rustfilt \ -show-instantiations \ - -output-dir=./target/coverage/html \ + -output-dir=./$CARGO_TARGET_DIR/coverage/html \ -ignore-filename-regex=".*/.cargo/registry/.*" \ -ignore-filename-regex=".*/.cargo/git/.*" \ -ignore-filename-regex=".*/.rustup/.*" \ - -instr-profile=./target/coverage/test.profdata \ - $(find target/ -type f -perm -u+x ! -name '*.so') + -instr-profile=./$CARGO_TARGET_DIR/coverage/test.profdata \ + $(printf -- "-object %s " $(cat ./$CARGO_TARGET_DIR/files.txt)) -xdg-open ./target/coverage/html/index.html +xdg-open ./$CARGO_TARGET_DIR/coverage/html/index.html From 8b255be076de697a05dea71dc8f143d1b10c3d2a Mon Sep 17 00:00:00 2001 From: Jane Lusby Date: Tue, 1 Dec 2020 14:33:18 -0800 Subject: [PATCH 6/8] remove sparse flag from coverage --- zebra-utils/coverage | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zebra-utils/coverage b/zebra-utils/coverage index 4c1b5955700..74f0ea77e0d 100755 --- a/zebra-utils/coverage +++ b/zebra-utils/coverage @@ -9,9 +9,9 @@ mkdir -p ./$CARGO_TARGET_DIR/coverage export ZEBRA_SKIP_NETWORK_TESTS=1 export LLVM_PROFILE_FILE="${PWD}/$CARGO_TARGET_DIR/coverage/test.%m.profraw" export RUSTFLAGS="-Zinstrument-coverage -C link-dead-code -C debuginfo=2" -cargo +nightly test --no-run --message-format=json | jq -r "select(.profile.test == true) | .filenames[]" | grep -v dSYM - > ./$CARGO_TARGET_DIR/files.txt cargo +nightly test -$(rustc +nightly --print target-libdir)/../bin/llvm-profdata merge --sparse ./$CARGO_TARGET_DIR/coverage/test.*.profraw -o ./$CARGO_TARGET_DIR/coverage/test.profdata +cargo +nightly test --no-run --message-format=json | jq -r "select(.profile.test == true) | .filenames[]" | grep -v dSYM - > ./$CARGO_TARGET_DIR/files.txt +$(rustc +nightly --print target-libdir)/../bin/llvm-profdata merge ./$CARGO_TARGET_DIR/coverage/test.*.profraw -o ./$CARGO_TARGET_DIR/coverage/test.profdata rm -rf ./$CARGO_TARGET_DIR/coverage/html/ From 78ed49b17d085934b5ee32a875ddf1b070af86df Mon Sep 17 00:00:00 2001 From: Jane Lusby Date: Tue, 1 Dec 2020 15:00:16 -0800 Subject: [PATCH 7/8] update the coverage script organization --- zebra-utils/coverage | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/zebra-utils/coverage b/zebra-utils/coverage index 74f0ea77e0d..9182a46daa9 100755 --- a/zebra-utils/coverage +++ b/zebra-utils/coverage @@ -3,12 +3,12 @@ set -e set -o xtrace export CARGO_TARGET_DIR="coverage-target" - -rm -rf ./"$CARGO_TARGET_DIR"/coverage -mkdir -p ./$CARGO_TARGET_DIR/coverage export ZEBRA_SKIP_NETWORK_TESTS=1 export LLVM_PROFILE_FILE="${PWD}/$CARGO_TARGET_DIR/coverage/test.%m.profraw" export RUSTFLAGS="-Zinstrument-coverage -C link-dead-code -C debuginfo=2" + +rm -rf ./"$CARGO_TARGET_DIR"/coverage +mkdir -p ./$CARGO_TARGET_DIR/coverage cargo +nightly test cargo +nightly test --no-run --message-format=json | jq -r "select(.profile.test == true) | .filenames[]" | grep -v dSYM - > ./$CARGO_TARGET_DIR/files.txt $(rustc +nightly --print target-libdir)/../bin/llvm-profdata merge ./$CARGO_TARGET_DIR/coverage/test.*.profraw -o ./$CARGO_TARGET_DIR/coverage/test.profdata @@ -26,4 +26,9 @@ $(rustc +nightly --print target-libdir)/../bin/llvm-cov show \ -instr-profile=./$CARGO_TARGET_DIR/coverage/test.profdata \ $(printf -- "-object %s " $(cat ./$CARGO_TARGET_DIR/files.txt)) +$(rustc +nightly --print target-libdir)/../bin/llvm-cov export \ + -format=lcov \ + -instr-profile=./$CARGO_TARGET_DIR/coverage/test.profdata \ + $(printf -- "-object %s " $(cat ./$CARGO_TARGET_DIR/files.txt)) > "./$CARG_TARGET_DIR/coverage/lcov.info" + xdg-open ./$CARGO_TARGET_DIR/coverage/html/index.html From eb4398baa1d12e81cbe2022a52d04865eb45fd8d Mon Sep 17 00:00:00 2001 From: Jane Lusby Date: Wed, 2 Dec 2020 11:09:24 -0800 Subject: [PATCH 8/8] fix typo in coverage script --- zebra-utils/coverage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-utils/coverage b/zebra-utils/coverage index 9182a46daa9..8af3fdc16bb 100755 --- a/zebra-utils/coverage +++ b/zebra-utils/coverage @@ -29,6 +29,6 @@ $(rustc +nightly --print target-libdir)/../bin/llvm-cov show \ $(rustc +nightly --print target-libdir)/../bin/llvm-cov export \ -format=lcov \ -instr-profile=./$CARGO_TARGET_DIR/coverage/test.profdata \ - $(printf -- "-object %s " $(cat ./$CARGO_TARGET_DIR/files.txt)) > "./$CARG_TARGET_DIR/coverage/lcov.info" + $(printf -- "-object %s " $(cat ./$CARGO_TARGET_DIR/files.txt)) > "./$CARGO_TARGET_DIR/coverage/lcov.info" xdg-open ./$CARGO_TARGET_DIR/coverage/html/index.html