From d2c770cd36952a084b498313442d1b026f4117e1 Mon Sep 17 00:00:00 2001 From: Levi Waldron Date: Thu, 5 Nov 2026 08:25:45 -0500 Subject: [PATCH 1/3] Upload testthat results to Codecov Test Analytics from R CMD check Co-authored-by: Claude Code --- .github/workflows/bioccheck.yml | 60 ++++++++++++++++++++++--- bioccheck_usage.md | 10 ++++- docs/adr/0014-codecov-test-analytics.md | 34 ++++++++++++++ docs/adr/README.md | 1 + docs/index.html | 2 +- 5 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 docs/adr/0014-codecov-test-analytics.md diff --git a/.github/workflows/bioccheck.yml b/.github/workflows/bioccheck.yml index 7791bae..6f55a84 100644 --- a/.github/workflows/bioccheck.yml +++ b/.github/workflows/bioccheck.yml @@ -139,20 +139,49 @@ jobs: remotes::install_local(".", dependencies = TRUE, Ncpus = parallel::detectCores()) # Only install the tools for features that are used in this run. pkgs <- c("rcmdcheck", "BiocCheck") - upload_coverage <- nzchar(Sys.getenv("CODECOV_TOKEN")) + upload_codecov <- nzchar(Sys.getenv("CODECOV_TOKEN")) report_coverage <- Sys.getenv("ENABLE_COVERAGE_REPORT") == "true" - if (upload_coverage || report_coverage) pkgs <- c(pkgs, "covr") - if (upload_coverage) pkgs <- c(pkgs, "xml2") + if (upload_codecov || report_coverage) pkgs <- c(pkgs, "covr") + # xml2 is used by covr::to_cobertura() and by testthat's JunitReporter. + if (upload_codecov) pkgs <- c(pkgs, "xml2") if (report_coverage) pkgs <- c(pkgs, "DT", "htmltools") if (Sys.getenv("INSTALL_PKGDOWN") == "true") pkgs <- c(pkgs, "pkgdown") if (Sys.getenv("ENABLE_CYCLOCOMP") == "true") pkgs <- c(pkgs, "cyclocomp") BiocManager::install(pkgs, Ncpus = parallel::detectCores()) + # R CMD check runs tests/testthat.R in an `R --vanilla` subprocess that + # reads no profile except the startup file R copies from + # share/R/tests-startup.R, so that is where the JUnit reporter for + # Codecov Test Analytics is set up. testthat's check reporter keeps + # writing to testthat.Rout as before (ADR 0014). + - name: Set Up JUnit Test Results for Codecov + if: env.CODECOV_TOKEN != '' + continue-on-error: true + shell: bash + run: | + startup="$(Rscript -e 'cat(R.home("share"))')/R/tests-startup.R" + cat >> "$startup" <<'EOF' + + # JunitReporter errors without xml2, which would fail the tests. + if (nzchar(Sys.getenv("TESTTHAT_JUNIT_FILE")) && nzchar(system.file(package = "xml2"))) { + setHook(packageEvent("testthat", "onLoad"), function(...) { + options(testthat.default_check_reporter = testthat::MultiReporter$new(list( + testthat::CheckReporter$new(), + testthat::JunitReporter$new(file = Sys.getenv("TESTTHAT_JUNIT_FILE")) + ))) + }) + } + EOF + - name: Run R CMD check env: RCMDCHECK_ERROR_ON: ${{ inputs.rcmdcheck_error_on }} shell: Rscript {0} run: | + # Read by the startup hook above. Set here, not from the github.workspace context, + # which is the host path in container jobs; absolute because tests run in + # check/.Rcheck/tests. + if (nzchar(Sys.getenv("CODECOV_TOKEN"))) Sys.setenv(TESTTHAT_JUNIT_FILE = file.path(getwd(), "junit.xml")) allowed <- c("never", "note", "warning", "error") error_on <- Sys.getenv("RCMDCHECK_ERROR_ON", unset = "warning") if (!error_on %in% allowed) stop("Invalid rcmdcheck_error_on value: '", error_on, "'. Must be one of: ", paste(allowed, collapse = ", ")) @@ -231,7 +260,10 @@ jobs: # Bioconductor container doesn't include. - name: Install gpg for Codecov id: gpg - if: steps.coverage.outcome == 'success' && env.CODECOV_TOKEN != '' + if: >- + !cancelled() + && env.CODECOV_TOKEN != '' + && (steps.coverage.outcome == 'success' || hashFiles('junit.xml') != '') continue-on-error: true shell: bash env: @@ -244,7 +276,7 @@ jobs: # The Codecov action, unlike covr::codecov(), accepts an organization's # global upload token as well as a repository token (ADR 0011). - name: Upload Coverage to Codecov - if: steps.gpg.outcome == 'success' + if: steps.coverage.outcome == 'success' && steps.gpg.outcome == 'success' continue-on-error: true uses: codecov/codecov-action@v7 with: @@ -254,6 +286,24 @@ jobs: disable_telem: true fail_ci_if_error: false + # Runs even when R CMD check failed, since failing tests are what + # Test Analytics reports. Only testthat packages produce junit.xml. + - name: Upload Test Results to Codecov + if: >- + !cancelled() + && env.CODECOV_TOKEN != '' + && steps.gpg.outcome == 'success' + && hashFiles('junit.xml') != '' + continue-on-error: true + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + report_type: test_results + files: junit.xml + disable_search: true + disable_telem: true + fail_ci_if_error: false + - name: Run Cyclocomp if: inputs.enable_cyclocomp == true continue-on-error: true diff --git a/bioccheck_usage.md b/bioccheck_usage.md index 1fe8e62..f9ca13d 100644 --- a/bioccheck_usage.md +++ b/bioccheck_usage.md @@ -36,7 +36,7 @@ The `rcmdcheck_error_on` input was named `error_on` before 2026-11; callers must Even if you use the Workflow Generator, you must perform these one-time repository settings to use optional features: -**For Codecov** (coverage is uploaded on pushes and PRs to non-release branches only): +**For Codecov** (coverage is uploaded on pushes and PRs to non-release branches only; test results on all branches): 1. **Obtain Codecov Token**: Log in to [Codecov](https://about.codecov.io/), link your repository, and copy the repository upload token. Alternatively, an organization admin can copy the organization's global upload token (organization settings in Codecov), which works for every repository in the organization; store it once as a GitHub organization secret named `CODECOV_TOKEN`. Paste only the token itself, not `CODECOV_TOKEN=...`. 2. **Set Repository Secret**: In your GitHub repository, navigate to **Settings** → **Secrets and variables** → **Actions** → **New repository secret**. Create a secret named `CODECOV_TOKEN` containing your token. @@ -53,6 +53,12 @@ Two independent options, which can be used together. Both skip release branches, Coverage runs the test suite a second time with instrumentation, so either option adds roughly the time of one more test run. +### Codecov Test Analytics + +When `CODECOV_TOKEN` is set, packages tested with `testthat` also upload their test results to [Codecov Test Analytics](https://docs.codecov.com/docs/test-analytics), which lists failing, slow and flaky tests. The results come from the `R CMD check` run itself, so the tests are not run again, and they are uploaded on every branch, including release branches, and also when `R CMD check` fails. Packages that use `RUnit` or plain `tests/*.R` scripts upload no test results. An upload problem never fails the check. + +This only works when `tests/testthat.R` calls `test_check()` without its own `reporter` argument, as the file written by `usethis::use_testthat()` does. + ### Using the Cyclocomp Artifact When `enable_cyclocomp` is true, the workflow uploads a `cyclocomp-results` CSV artifact. @@ -118,7 +124,7 @@ Here's a full set of badges for the top of your package's `README.md`. Replace ` | `rcmdcheck_error_on` | `string` | `"warning"` | `R CMD check` failure policy (`"never"`, `"note"`, `"warning"`, `"error"`). | | `bioccheck_error_on` | `string` | `"error"` | `BiocCheck` failure policy (`"error"`, `"never"`). | | `bioc_version` | `string` | `""` | Optional override for container tag (e.g. `devel`, `RELEASE_3_20`). | -| `secrets.CODECOV_TOKEN` | secret | `""` | Optional Codecov token (repository or organization global upload token). Uploads coverage to Codecov when set. | +| `secrets.CODECOV_TOKEN` | secret | `""` | Optional Codecov token (repository or organization global upload token). Uploads coverage, and `testthat` test results for [Test Analytics](#codecov-test-analytics), to Codecov when set. | ## Which Version to Use diff --git a/docs/adr/0014-codecov-test-analytics.md b/docs/adr/0014-codecov-test-analytics.md new file mode 100644 index 0000000..600a5aa --- /dev/null +++ b/docs/adr/0014-codecov-test-analytics.md @@ -0,0 +1,34 @@ +# 0014. Codecov Test Analytics from the `R CMD check` Run + +- **Status:** Proposed +- **Date:** 2026-11-05 +- **Deciders:** Levi Waldron + +## Context and Problem Statement + +[Codecov Test Analytics](https://docs.codecov.com/docs/test-analytics) reports failing, slow and flaky tests from JUnit XML files uploaded with `codecov/codecov-action` (`report_type: test_results`). `testthat` can write JUnit XML with `JunitReporter`, but `bioccheck.yml` runs the tests inside `R CMD check`, which uses `testthat`'s check reporter. Running the tests a third time only to write JUnit XML (they already run in `R CMD check` and again under `covr`) would add a full test run to every job. + +`test_check()` takes its default reporter from the `testthat.default_check_reporter` option. `R CMD check` runs `tests/testthat.R` with `R --vanilla`, which reads no user or site profile, so that option cannot be set with `.Rprofile` or `Rprofile.site`. The only startup code that subprocess runs is the file `R CMD check` copies from `$(R RHOME)/share/R/tests-startup.R` and passes as `R_TESTS`. + +## Decision + +When `CODECOV_TOKEN` is set, a step before `R CMD check` appends code to `share/R/tests-startup.R` in the job's container. If `TESTTHAT_JUNIT_FILE` is set and `xml2` (which `JunitReporter` needs) is installed, that code registers an `onLoad` hook for `testthat` which sets `testthat.default_check_reporter` to a `MultiReporter` of the usual `CheckReporter` and a `JunitReporter` writing to that file. The `R CMD check` step sets `TESTTHAT_JUNIT_FILE` with `Sys.setenv()` to an absolute path in the workspace, so the `covr` step, which does not set it, does not overwrite the file. It is set in R because `${{ github.workspace }}` is the runner's host path in container jobs. + +A new step uploads the file with `codecov/codecov-action@v7` and `report_type: test_results`. It runs on every branch, including release branches, and also when `R CMD check` fails, because failing tests are what Test Analytics reports. The `gpg` step now runs when either coverage or test results are to be uploaded. As with coverage (ADR 0011), the setup and upload steps use `continue-on-error`, so test results never fail the check. + +No new input is added: the `CODECOV_TOKEN` secret remains the opt-in for everything sent to Codecov (ADR 0005). + +## Alternatives Considered + +- **Write JUnit XML from the `covr` run**: needs no change to R's files, but `covr` runs only after `R CMD check` succeeds and only on non-release branches, so failing tests would never be reported. +- **Run `testthat::test_local()` with `JunitReporter` in its own step**: simple, but runs the whole test suite once more. +- **Patch the package's `tests/testthat.R` before checking**: changes the package being checked and the tarball `BiocCheck` reads. +- **A separate `enable_test_analytics` input**: more configuration for callers, when uploading to Codecov is already opted into with the token. + +## Consequences + +- Packages using `testthat` get Test Analytics with no change to their own files and no extra test run. +- The normal `testthat` output in `testthat.Rout`, and the failure of `R CMD check` on failing tests, are unchanged. This also holds for packages that run their tests in parallel (`Config/testthat/parallel: true`), since `MultiReporter` supports parallel mode; there `testthat` 3.3.2 writes `classname="character(0)"` in the JUnit file, which is an upstream quirk. +- Packages using `RUnit` or plain `tests/*.R` scripts, or whose `tests/testthat.R` passes its own `reporter`, upload no test results. +- The approach depends on `R CMD check` copying `share/R/tests-startup.R` into the tests directory, which is an internal detail of R. If R changes it, `junit.xml` is no longer written and the upload is skipped; the check itself is unaffected. +- Runs on release branches, and runs where `R CMD check` fails, now spend about 20 seconds installing `gnupg` when there are test results to upload. diff --git a/docs/adr/README.md b/docs/adr/README.md index 31af938..ca24054 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -17,3 +17,4 @@ This directory contains Architecture Decision Records (ADRs) for the `bioconduct - [0011. Upload Coverage with the Codecov Action](0011-codecov-action-upload.md) - [0012. Opt-in Coverage Report Without Codecov](0012-opt-in-coverage-report.md) - [0013. Separate Failure Policies for `R CMD check` and `BiocCheck`](0013-separate-bioccheck-failure-policy.md) +- [0014. Codecov Test Analytics from the `R CMD check` Run](0014-codecov-test-analytics.md) diff --git a/docs/index.html b/docs/index.html index 0798dc7..d980877 100644 --- a/docs/index.html +++ b/docs/index.html @@ -304,7 +304,7 @@

Configuration

Enable Codecov - Uploads test coverage to Codecov using covr. Requires a CODECOV_TOKEN repository or organization secret. + Uploads test coverage (covr) and testthat test results to Codecov. Requires a CODECOV_TOKEN repository or organization secret.
From ebe1b625e8b9c2c9f11acfa9898ab072e7128d55 Mon Sep 17 00:00:00 2001 From: Levi Waldron Date: Thu, 5 Nov 2026 08:29:34 -0500 Subject: [PATCH 2/3] accepted ADR 0014 --- docs/adr/0014-codecov-test-analytics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adr/0014-codecov-test-analytics.md b/docs/adr/0014-codecov-test-analytics.md index 600a5aa..399c4f4 100644 --- a/docs/adr/0014-codecov-test-analytics.md +++ b/docs/adr/0014-codecov-test-analytics.md @@ -1,6 +1,6 @@ # 0014. Codecov Test Analytics from the `R CMD check` Run -- **Status:** Proposed +- **Status:** Accepted - **Date:** 2026-11-05 - **Deciders:** Levi Waldron From 150e952acbda6e79139e81596490b8861bbbc7f6 Mon Sep 17 00:00:00 2001 From: Levi Waldron Date: Thu, 5 Nov 2026 10:43:23 -0500 Subject: [PATCH 3/3] Upload only junit.xml written by this run's R CMD check Co-authored-by: Claude Code --- .github/workflows/bioccheck.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bioccheck.yml b/.github/workflows/bioccheck.yml index 6f55a84..1e2fbcc 100644 --- a/.github/workflows/bioccheck.yml +++ b/.github/workflows/bioccheck.yml @@ -174,6 +174,7 @@ jobs: EOF - name: Run R CMD check + id: rcmdcheck env: RCMDCHECK_ERROR_ON: ${{ inputs.rcmdcheck_error_on }} shell: Rscript {0} @@ -181,7 +182,12 @@ jobs: # Read by the startup hook above. Set here, not from the github.workspace context, # which is the host path in container jobs; absolute because tests run in # check/.Rcheck/tests. - if (nzchar(Sys.getenv("CODECOV_TOKEN"))) Sys.setenv(TESTTHAT_JUNIT_FILE = file.path(getwd(), "junit.xml")) + # Removing any junit.xml from the checkout means only this run's results are uploaded. + if (nzchar(Sys.getenv("CODECOV_TOKEN"))) { + junit_file <- file.path(getwd(), "junit.xml") + unlink(junit_file) + Sys.setenv(TESTTHAT_JUNIT_FILE = junit_file) + } allowed <- c("never", "note", "warning", "error") error_on <- Sys.getenv("RCMDCHECK_ERROR_ON", unset = "warning") if (!error_on %in% allowed) stop("Invalid rcmdcheck_error_on value: '", error_on, "'. Must be one of: ", paste(allowed, collapse = ", ")) @@ -263,7 +269,10 @@ jobs: if: >- !cancelled() && env.CODECOV_TOKEN != '' - && (steps.coverage.outcome == 'success' || hashFiles('junit.xml') != '') + && ( + steps.coverage.outcome == 'success' + || (steps.rcmdcheck.outcome != 'skipped' && hashFiles('junit.xml') != '') + ) continue-on-error: true shell: bash env: @@ -293,6 +302,7 @@ jobs: !cancelled() && env.CODECOV_TOKEN != '' && steps.gpg.outcome == 'success' + && steps.rcmdcheck.outcome != 'skipped' && hashFiles('junit.xml') != '' continue-on-error: true uses: codecov/codecov-action@v7