Report bare expectations (#498)

* sketch for reporting bare expectations

* optional wrap argument

* use NULL as test description for wrapped tests

to avoid nested start_test() and end_test() calls

* only wrap from test_files()

* output

* wrap = TRUE is the default

* NEWS

* document wrap argument to test_file()

* only call skip_empty() if not wrapped

* adapt encoding tests

* document

* markdown

* adapt tests

NEWS.md

@@ -71,6 +71,8 @@
 
 * New argument `load_helpers` in `test_dir()` (#505).
 
+* Bare expectations notify the reporter again. This is achieved by running all tests inside `test_code()` by default (#427, #498).
+
 * New `DebugReporter` that calls a better version of `recover()` in case of failures, errors, or warnings (#360, #470).
 
 * `compare.numeric()` respects `check.attributes()` so `expect_equivalent()`
@@ -100,7 +102,7 @@
 
 # testthat 1.0.2
 
-* Ensure 'std::logic_error()' constructed with 'std::string()'
+* Ensure `std::logic_error()` constructed with `std::string()`
   argument, to avoid build errors on Solaris.
 
 # testthat 1.0.1

R/source.R

@@ -9,8 +9,10 @@
 #' @param env Environment in which to evaluate code.
 #' @param chdir Change working directory to `dirname(path)`?
 #' @param encoding File encoding, default: "unknown"
+#' @param wrap Add a [test_that()] call around the code?
 #' @export
-source_file <- function(path, env = test_env(), chdir = TRUE, encoding = "unknown") {
+source_file <- function(path, env = test_env(), chdir = TRUE,
+                        encoding = "unknown", wrap = TRUE) {
   stopifnot(file.exists(path))
   stopifnot(is.environment(env))
 
@@ -25,19 +27,23 @@ source_file <- function(path, env = test_env(), chdir = TRUE, encoding = "unknow
     old_dir <- setwd(dirname(path))
     on.exit(setwd(old_dir), add = TRUE)
   }
-  invisible(eval(exprs, env))
+  if (wrap) {
+    invisible(test_code(NULL, exprs, env))
+  } else {
+    invisible(eval(exprs, env))
+  }
 }
 
 #' @rdname source_file
 #' @export
 source_dir <- function(path, pattern = "\\.[rR]$", env = test_env(),
-                       chdir = TRUE) {
+                       chdir = TRUE, wrap = TRUE) {
   files <- normalizePath(sort(dir(path, pattern, full.names = TRUE)))
-  lapply(files, source_file, env = env, chdir = chdir)
+  lapply(files, source_file, env = env, chdir = chdir, wrap = wrap)
 }
 
 #' @rdname source_file
 #' @export
 source_test_helpers <- function(path = "tests/testthat", env = test_env()) {
-  source_dir(path, "^helper.*\\.[rR]$", env = env)
+  source_dir(path, "^helper.*\\.[rR]$", env = env, wrap = FALSE)
 }

R/test-files.R

@@ -52,7 +52,8 @@ test_files <- function(paths, reporter = default_reporter(), env = test_env(),
       reporter = current_reporter,
       start_end_reporter = FALSE,
       load_helpers = FALSE,
-      encoding = encoding
+      encoding = encoding,
+      wrap = TRUE
     )
   )
 
@@ -102,11 +103,12 @@ find_test_scripts <- function(path, filter = NULL, invert = FALSE, ...) {
 #' @param encoding File encoding, default is "unknown"
 #' `unknown`.
 #' @inheritParams with_reporter
+#' @inheritParams source_file
 #' @return the results as a "testthat_results" (list)
 #' @export
 test_file <- function(path, reporter = default_reporter(), env = test_env(),
                       start_end_reporter = TRUE, load_helpers = TRUE,
-                      encoding = "unknown") {
+                      encoding = "unknown", wrap = TRUE) {
   library(testthat)
 
   reporter <- find_reporter(reporter)
@@ -134,7 +136,7 @@ test_file <- function(path, reporter = default_reporter(), env = test_env(),
       lister$start_file(basename(path))
 
       source_file(path, new.env(parent = env),
-                  chdir = TRUE, encoding = encoding)
+                  chdir = TRUE, encoding = encoding, wrap = wrap)
 
       end_context()
     }

R/test-that.R

@@ -32,8 +32,10 @@ test_that <- function(desc, code) {
 }
 
 test_code <- function(test, code, env = test_env(), skip_on_empty = TRUE) {
-  get_reporter()$start_test(context = get_reporter()$.context, test = test)
-  on.exit(get_reporter()$end_test(context = get_reporter()$.context, test = test))
+  if (!is.null(test)) {
+    get_reporter()$start_test(context = get_reporter()$.context, test = test)
+    on.exit(get_reporter()$end_test(context = get_reporter()$.context, test = test))
+  }
 
   ok <- TRUE
   register_expectation <- function(e) {
@@ -147,7 +149,7 @@ test_code <- function(test, code, env = test_env(), skip_on_empty = TRUE) {
   tryCatch(
     withCallingHandlers({
       eval(code, test_env)
-      if (!handled)
+      if (!handled && !is.null(test))
         skip_empty()
       },
       expectation = handle_expectation,

tests/testthat/reporters/check.txt

@@ -40,6 +40,8 @@ evaluation nested too deeply: infinite recursion / options(expressions=)?
 9: f() at reporters/tests.R:42
 10: f() at reporters/tests.R:42
 ...
+164: f() at reporters/tests.R:42
+165: f() at reporters/tests.R:42
 166: f() at reporters/tests.R:42
 167: f() at reporters/tests.R:42
 168: f() at reporters/tests.R:42
@@ -48,8 +50,6 @@ evaluation nested too deeply: infinite recursion / options(expressions=)?
 171: f() at reporters/tests.R:42
 172: f() at reporters/tests.R:42
 173: f() at reporters/tests.R:42
-174: f() at reporters/tests.R:42
-175: f() at reporters/tests.R:42
 
 testthat results ================================================================
 OK: 7 SKIPPED: 3 FAILED: 7

tests/testthat/reporters/debug.txt

@@ -187,8 +187,6 @@
 171: tests.R#42: f()
 172: tests.R#42: f()
 173: tests.R#42: f()
-174: tests.R#42: f()
-175: tests.R#42: f()
 
 1: tests.R#49: skip("skip")
 

tests/testthat/reporters/junit.txt

@@ -49,16 +49,16 @@
 9: f() at reporters/tests.R:42
 10: f() at reporters/tests.R:42
 ...
+164: f() at reporters/tests.R:42
+165: f() at reporters/tests.R:42
 166: f() at reporters/tests.R:42
 167: f() at reporters/tests.R:42
 168: f() at reporters/tests.R:42
 169: f() at reporters/tests.R:42
 170: f() at reporters/tests.R:42
 171: f() at reporters/tests.R:42
 172: f() at reporters/tests.R:42
-173: f() at reporters/tests.R:42
-174: f() at reporters/tests.R:42
-175: f() at reporters/tests.R:42</error>
+173: f() at reporters/tests.R:42</error>
     </testcase>
   </testsuite>
   <testsuite name="Skips" timestamp="1999:12:31 23:59:59" hostname="nodename" tests="3" skipped="3" failures="0" errors="0" time="0">
@@ -81,4 +81,4 @@
     <testcase time="0" classname="Output" name="Output:1"/>
     <testcase time="0" classname="Output" name="Output:1"/>
   </testsuite>
-</testsuites>
+</testsuites>

tests/testthat/reporters/summary-no-dots.txt

@@ -62,6 +62,8 @@ evaluation nested too deeply: infinite recursion / options(expressions=)?
 9: f() at reporters/tests.R:42
 10: f() at reporters/tests.R:42
 ...
+164: f() at reporters/tests.R:42
+165: f() at reporters/tests.R:42
 166: f() at reporters/tests.R:42
 167: f() at reporters/tests.R:42
 168: f() at reporters/tests.R:42
@@ -70,7 +72,5 @@ evaluation nested too deeply: infinite recursion / options(expressions=)?
 171: f() at reporters/tests.R:42
 172: f() at reporters/tests.R:42
 173: f() at reporters/tests.R:42
-174: f() at reporters/tests.R:42
-175: f() at reporters/tests.R:42
 
 DONE ===========================================================================

tests/testthat/reporters/summary.txt

@@ -62,6 +62,8 @@ evaluation nested too deeply: infinite recursion / options(expressions=)?
 9: f() at reporters/tests.R:42
 10: f() at reporters/tests.R:42
 ...
+164: f() at reporters/tests.R:42
+165: f() at reporters/tests.R:42
 166: f() at reporters/tests.R:42
 167: f() at reporters/tests.R:42
 168: f() at reporters/tests.R:42
@@ -70,7 +72,5 @@ evaluation nested too deeply: infinite recursion / options(expressions=)?
 171: f() at reporters/tests.R:42
 172: f() at reporters/tests.R:42
 173: f() at reporters/tests.R:42
-174: f() at reporters/tests.R:42
-175: f() at reporters/tests.R:42
 
 DONE ===========================================================================

tests/testthat/reporters/tap.txt

@@ -40,6 +40,8 @@ not ok 9 Recursion:1
   9: f() at reporters/tests.R:42
   10: f() at reporters/tests.R:42
   ...
+  164: f() at reporters/tests.R:42
+  165: f() at reporters/tests.R:42
   166: f() at reporters/tests.R:42
   167: f() at reporters/tests.R:42
   168: f() at reporters/tests.R:42
@@ -48,8 +50,6 @@ not ok 9 Recursion:1
   171: f() at reporters/tests.R:42
   172: f() at reporters/tests.R:42
   173: f() at reporters/tests.R:42
-  174: f() at reporters/tests.R:42
-  175: f() at reporters/tests.R:42
 # Context Skips
 ok 10 # SKIP skip
 ok 11 # SKIP skip

tests/testthat/reporters/teamcity.txt

@@ -52,7 +52,7 @@
 ##teamcity[testSuiteStarted name='Recursion']
 ##teamcity[testSuiteStarted name='Recursion:1']
 ##teamcity[testStarted name='expectation 1']
-##teamcity[testFailed name='expectation 1' message='evaluation nested too deeply: infinite recursion / options(expressions=)?' details='1: f() at reporters/tests.R:43|n2: f() at reporters/tests.R:42|n3: f() at reporters/tests.R:42|n4: f() at reporters/tests.R:42|n5: f() at reporters/tests.R:42|n6: f() at reporters/tests.R:42|n7: f() at reporters/tests.R:42|n8: f() at reporters/tests.R:42|n9: f() at reporters/tests.R:42|n10: f() at reporters/tests.R:42|n...|n166: f() at reporters/tests.R:42|n167: f() at reporters/tests.R:42|n168: f() at reporters/tests.R:42|n169: f() at reporters/tests.R:42|n170: f() at reporters/tests.R:42|n171: f() at reporters/tests.R:42|n172: f() at reporters/tests.R:42|n173: f() at reporters/tests.R:42|n174: f() at reporters/tests.R:42|n175: f() at reporters/tests.R:42']
+##teamcity[testFailed name='expectation 1' message='evaluation nested too deeply: infinite recursion / options(expressions=)?' details='1: f() at reporters/tests.R:43|n2: f() at reporters/tests.R:42|n3: f() at reporters/tests.R:42|n4: f() at reporters/tests.R:42|n5: f() at reporters/tests.R:42|n6: f() at reporters/tests.R:42|n7: f() at reporters/tests.R:42|n8: f() at reporters/tests.R:42|n9: f() at reporters/tests.R:42|n10: f() at reporters/tests.R:42|n...|n164: f() at reporters/tests.R:42|n165: f() at reporters/tests.R:42|n166: f() at reporters/tests.R:42|n167: f() at reporters/tests.R:42|n168: f() at reporters/tests.R:42|n169: f() at reporters/tests.R:42|n170: f() at reporters/tests.R:42|n171: f() at reporters/tests.R:42|n172: f() at reporters/tests.R:42|n173: f() at reporters/tests.R:42']
 ##teamcity[testFinished name='expectation 1']
 ##teamcity[testSuiteFinished name='Recursion:1']
 

tests/testthat/test-context.R

@@ -28,7 +28,7 @@ CountReporter <- R6::R6Class("CountReporter", inherit = Reporter,
 
 test_that("contexts are opened, then closed", {
   report <- CountReporter$new()
-  test_file("context.R", report)
+  test_file("context.R", report, wrap = FALSE)
   expect_that(report$context_count, equals(2))
   expect_that(report$context_i, equals(0))
   expect_that(report$test_count, equals(4))

tests/testthat/test-encoding.R

@@ -1,17 +1,17 @@
 context("encoding")
 
 test_that("can source file with Latin-1 encoding", {
-  expect_error(test_file("latin1.R", "stop", encoding = "latin1"), NA)
+  expect_error(test_file("latin1.R", "stop", encoding = "latin1", wrap = FALSE), NA)
 })
 
 test_that("can source file with UTF-8 encoding", {
-  expect_error(test_file("utf8.R", "stop", encoding = "UTF-8"), NA)
+  expect_error(test_file("utf8.R", "stop", encoding = "UTF-8", wrap = FALSE), NA)
 })
 
 test_that("error with Latin-1 encoding mismatch", {
-  expect_error(test_file("latin1.R", "stop", encoding = "UTF-8"))
+  expect_error(test_file("latin1.R", "stop", encoding = "UTF-8", wrap = FALSE))
 })
 
 test_that("error with UTF-8 encoding mismatch", {
-  expect_error(test_file("utf8.R", "stop", encoding = "latin1"))
+  expect_error(test_file("utf8.R", "stop", encoding = "latin1", wrap = FALSE))
 })

tests/testthat/test-reporter.R

@@ -49,7 +49,7 @@ test_that("reporters produce consistent output", {
       expect_error(
         withr::with_options(
           list(expressions = Cstack_info()[["eval_depth"]] + 200),
-          test_file(test_path("reporters/tests.R"), reporter)
+          test_file(test_path("reporters/tests.R"), reporter, wrap = FALSE)
         ),
         error_regexp
       ),

tests/testthat/test-source_dir.R

@@ -1,15 +1,15 @@
 context("source_dir")
 
 test_that('source_dir()', {
-  res <- source_dir('test_dir', pattern = 'hello', chdir = TRUE)
+  res <- source_dir('test_dir', pattern = 'hello', chdir = TRUE, wrap = FALSE)
   expect_equal(res[[1]](), "Hello World")
-
-  res <- source_dir(normalizePath('test_dir'), pattern = 'hello', chdir = TRUE)
+
+  res <- source_dir(normalizePath('test_dir'), pattern = 'hello', chdir = TRUE, wrap = FALSE)
+  expect_equal(res[[1]](), "Hello World")
+
+  res <- source_dir('test_dir', pattern = 'hello', chdir = FALSE, wrap = FALSE)
   expect_equal(res[[1]](), "Hello World")
 
-  res <- source_dir('test_dir', pattern = 'hello', chdir = FALSE)
-  expect_equal(res[[1]](), "Hello World")   
-
-  res <- source_dir(normalizePath('test_dir'), pattern = 'hello', chdir = FALSE)
+  res <- source_dir(normalizePath('test_dir'), pattern = 'hello', chdir = FALSE, wrap = FALSE)
   expect_equal(res[[1]](), "Hello World")
 })