From 37cc0d04fc497edcf62fe44729c73e3c77559b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Mon, 20 Feb 2017 16:53:01 +0100 Subject: [PATCH] Report bare expectations (#498) * sketch for reporting bare expectations * optional wrap argument * use NULL as test description for wrapped tests to avoid nested start_test() and end_test() calls * only wrap from test_files() * output * wrap = TRUE is the default * NEWS * document wrap argument to test_file() * only call skip_empty() if not wrapped * adapt encoding tests * document * markdown * adapt tests --- NEWS.md | 4 +++- R/source.R | 16 +++++++++++----- R/test-files.R | 8 +++++--- R/test-that.R | 8 +++++--- man/source_file.Rd | 8 ++++++-- man/test_file.Rd | 5 ++++- tests/testthat/reporters/check.txt | 4 ++-- tests/testthat/reporters/debug.txt | 2 -- tests/testthat/reporters/junit.txt | 8 ++++---- tests/testthat/reporters/summary-no-dots.txt | 4 ++-- tests/testthat/reporters/summary.txt | 4 ++-- tests/testthat/reporters/tap.txt | 4 ++-- tests/testthat/reporters/teamcity.txt | 2 +- tests/testthat/test-context.R | 2 +- tests/testthat/test-encoding.R | 8 ++++---- tests/testthat/test-reporter.R | 2 +- tests/testthat/test-source_dir.R | 14 +++++++------- 17 files changed, 60 insertions(+), 43 deletions(-) diff --git a/NEWS.md b/NEWS.md index 35ae995cf..e7b75c25e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -71,6 +71,8 @@ * New argument `load_helpers` in `test_dir()` (#505). +* Bare expectations notify the reporter again. This is achieved by running all tests inside `test_code()` by default (#427, #498). + * New `DebugReporter` that calls a better version of `recover()` in case of failures, errors, or warnings (#360, #470). * `compare.numeric()` respects `check.attributes()` so `expect_equivalent()` @@ -100,7 +102,7 @@ # testthat 1.0.2 -* Ensure 'std::logic_error()' constructed with 'std::string()' +* Ensure `std::logic_error()` constructed with `std::string()` argument, to avoid build errors on Solaris. # testthat 1.0.1 diff --git a/R/source.R b/R/source.R index e24086757..afc7bcdc7 100644 --- a/R/source.R +++ b/R/source.R @@ -9,8 +9,10 @@ #' @param env Environment in which to evaluate code. #' @param chdir Change working directory to `dirname(path)`? #' @param encoding File encoding, default: "unknown" +#' @param wrap Add a [test_that()] call around the code? #' @export -source_file <- function(path, env = test_env(), chdir = TRUE, encoding = "unknown") { +source_file <- function(path, env = test_env(), chdir = TRUE, + encoding = "unknown", wrap = TRUE) { stopifnot(file.exists(path)) stopifnot(is.environment(env)) @@ -25,19 +27,23 @@ source_file <- function(path, env = test_env(), chdir = TRUE, encoding = "unknow old_dir <- setwd(dirname(path)) on.exit(setwd(old_dir), add = TRUE) } - invisible(eval(exprs, env)) + if (wrap) { + invisible(test_code(NULL, exprs, env)) + } else { + invisible(eval(exprs, env)) + } } #' @rdname source_file #' @export source_dir <- function(path, pattern = "\\.[rR]$", env = test_env(), - chdir = TRUE) { + chdir = TRUE, wrap = TRUE) { files <- normalizePath(sort(dir(path, pattern, full.names = TRUE))) - lapply(files, source_file, env = env, chdir = chdir) + lapply(files, source_file, env = env, chdir = chdir, wrap = wrap) } #' @rdname source_file #' @export source_test_helpers <- function(path = "tests/testthat", env = test_env()) { - source_dir(path, "^helper.*\\.[rR]$", env = env) + source_dir(path, "^helper.*\\.[rR]$", env = env, wrap = FALSE) } diff --git a/R/test-files.R b/R/test-files.R index 68f9a44c3..ae36667de 100644 --- a/R/test-files.R +++ b/R/test-files.R @@ -52,7 +52,8 @@ test_files <- function(paths, reporter = default_reporter(), env = test_env(), reporter = current_reporter, start_end_reporter = FALSE, load_helpers = FALSE, - encoding = encoding + encoding = encoding, + wrap = TRUE ) ) @@ -102,11 +103,12 @@ find_test_scripts <- function(path, filter = NULL, invert = FALSE, ...) { #' @param encoding File encoding, default is "unknown" #' `unknown`. #' @inheritParams with_reporter +#' @inheritParams source_file #' @return the results as a "testthat_results" (list) #' @export test_file <- function(path, reporter = default_reporter(), env = test_env(), start_end_reporter = TRUE, load_helpers = TRUE, - encoding = "unknown") { + encoding = "unknown", wrap = TRUE) { library(testthat) reporter <- find_reporter(reporter) @@ -134,7 +136,7 @@ test_file <- function(path, reporter = default_reporter(), env = test_env(), lister$start_file(basename(path)) source_file(path, new.env(parent = env), - chdir = TRUE, encoding = encoding) + chdir = TRUE, encoding = encoding, wrap = wrap) end_context() } diff --git a/R/test-that.R b/R/test-that.R index ce78574fc..05d45cfc2 100644 --- a/R/test-that.R +++ b/R/test-that.R @@ -32,8 +32,10 @@ test_that <- function(desc, code) { } test_code <- function(test, code, env = test_env(), skip_on_empty = TRUE) { - get_reporter()$start_test(context = get_reporter()$.context, test = test) - on.exit(get_reporter()$end_test(context = get_reporter()$.context, test = test)) + if (!is.null(test)) { + get_reporter()$start_test(context = get_reporter()$.context, test = test) + on.exit(get_reporter()$end_test(context = get_reporter()$.context, test = test)) + } ok <- TRUE register_expectation <- function(e) { @@ -147,7 +149,7 @@ test_code <- function(test, code, env = test_env(), skip_on_empty = TRUE) { tryCatch( withCallingHandlers({ eval(code, test_env) - if (!handled) + if (!handled && !is.null(test)) skip_empty() }, expectation = handle_expectation, diff --git a/man/source_file.Rd b/man/source_file.Rd index de02e8b2e..3cd27a888 100644 --- a/man/source_file.Rd +++ b/man/source_file.Rd @@ -6,9 +6,11 @@ \alias{source_test_helpers} \title{Source a file, directory, or all helpers.} \usage{ -source_file(path, env = test_env(), chdir = TRUE, encoding = "unknown") +source_file(path, env = test_env(), chdir = TRUE, encoding = "unknown", + wrap = TRUE) -source_dir(path, pattern = "\\\\.[rR]$", env = test_env(), chdir = TRUE) +source_dir(path, pattern = "\\\\.[rR]$", env = test_env(), chdir = TRUE, + wrap = TRUE) source_test_helpers(path = "tests/testthat", env = test_env()) } @@ -21,6 +23,8 @@ source_test_helpers(path = "tests/testthat", env = test_env()) \item{encoding}{File encoding, default: "unknown"} +\item{wrap}{Add a \code{\link[=test_that]{test_that()}} call around the code?} + \item{pattern}{Regular expression used to filter files} } \description{ diff --git a/man/test_file.Rd b/man/test_file.Rd index e9efa9ea2..5551889e5 100644 --- a/man/test_file.Rd +++ b/man/test_file.Rd @@ -5,7 +5,8 @@ \title{Run all tests in specified file.} \usage{ test_file(path, reporter = default_reporter(), env = test_env(), - start_end_reporter = TRUE, load_helpers = TRUE, encoding = "unknown") + start_end_reporter = TRUE, load_helpers = TRUE, encoding = "unknown", + wrap = TRUE) } \arguments{ \item{path}{path to file} @@ -20,6 +21,8 @@ test_file(path, reporter = default_reporter(), env = test_env(), \item{encoding}{File encoding, default is "unknown" \code{unknown}.} + +\item{wrap}{Add a \code{\link[=test_that]{test_that()}} call around the code?} } \value{ the results as a "testthat_results" (list) diff --git a/tests/testthat/reporters/check.txt b/tests/testthat/reporters/check.txt index 42328fb3a..f36ea2041 100644 --- a/tests/testthat/reporters/check.txt +++ b/tests/testthat/reporters/check.txt @@ -40,6 +40,8 @@ evaluation nested too deeply: infinite recursion / options(expressions=)? 9: f() at reporters/tests.R:42 10: f() at reporters/tests.R:42 ... +164: f() at reporters/tests.R:42 +165: f() at reporters/tests.R:42 166: f() at reporters/tests.R:42 167: f() at reporters/tests.R:42 168: f() at reporters/tests.R:42 @@ -48,8 +50,6 @@ evaluation nested too deeply: infinite recursion / options(expressions=)? 171: f() at reporters/tests.R:42 172: f() at reporters/tests.R:42 173: f() at reporters/tests.R:42 -174: f() at reporters/tests.R:42 -175: f() at reporters/tests.R:42 testthat results ================================================================ OK: 7 SKIPPED: 3 FAILED: 7 diff --git a/tests/testthat/reporters/debug.txt b/tests/testthat/reporters/debug.txt index 95dc08ca2..5457f2d99 100644 --- a/tests/testthat/reporters/debug.txt +++ b/tests/testthat/reporters/debug.txt @@ -187,8 +187,6 @@ 171: tests.R#42: f() 172: tests.R#42: f() 173: tests.R#42: f() -174: tests.R#42: f() -175: tests.R#42: f() 1: tests.R#49: skip("skip") diff --git a/tests/testthat/reporters/junit.txt b/tests/testthat/reporters/junit.txt index 25e4cb48b..8b3fbf783 100644 --- a/tests/testthat/reporters/junit.txt +++ b/tests/testthat/reporters/junit.txt @@ -49,6 +49,8 @@ 9: f() at reporters/tests.R:42 10: f() at reporters/tests.R:42 ... +164: f() at reporters/tests.R:42 +165: f() at reporters/tests.R:42 166: f() at reporters/tests.R:42 167: f() at reporters/tests.R:42 168: f() at reporters/tests.R:42 @@ -56,9 +58,7 @@ 170: f() at reporters/tests.R:42 171: f() at reporters/tests.R:42 172: f() at reporters/tests.R:42 -173: f() at reporters/tests.R:42 -174: f() at reporters/tests.R:42 -175: f() at reporters/tests.R:42 +173: f() at reporters/tests.R:42 @@ -81,4 +81,4 @@ - \ No newline at end of file + diff --git a/tests/testthat/reporters/summary-no-dots.txt b/tests/testthat/reporters/summary-no-dots.txt index 9131bb36c..cc0491def 100644 --- a/tests/testthat/reporters/summary-no-dots.txt +++ b/tests/testthat/reporters/summary-no-dots.txt @@ -62,6 +62,8 @@ evaluation nested too deeply: infinite recursion / options(expressions=)? 9: f() at reporters/tests.R:42 10: f() at reporters/tests.R:42 ... +164: f() at reporters/tests.R:42 +165: f() at reporters/tests.R:42 166: f() at reporters/tests.R:42 167: f() at reporters/tests.R:42 168: f() at reporters/tests.R:42 @@ -70,7 +72,5 @@ evaluation nested too deeply: infinite recursion / options(expressions=)? 171: f() at reporters/tests.R:42 172: f() at reporters/tests.R:42 173: f() at reporters/tests.R:42 -174: f() at reporters/tests.R:42 -175: f() at reporters/tests.R:42 DONE =========================================================================== diff --git a/tests/testthat/reporters/summary.txt b/tests/testthat/reporters/summary.txt index ce39b1ac1..ce6026ae4 100644 --- a/tests/testthat/reporters/summary.txt +++ b/tests/testthat/reporters/summary.txt @@ -62,6 +62,8 @@ evaluation nested too deeply: infinite recursion / options(expressions=)? 9: f() at reporters/tests.R:42 10: f() at reporters/tests.R:42 ... +164: f() at reporters/tests.R:42 +165: f() at reporters/tests.R:42 166: f() at reporters/tests.R:42 167: f() at reporters/tests.R:42 168: f() at reporters/tests.R:42 @@ -70,7 +72,5 @@ evaluation nested too deeply: infinite recursion / options(expressions=)? 171: f() at reporters/tests.R:42 172: f() at reporters/tests.R:42 173: f() at reporters/tests.R:42 -174: f() at reporters/tests.R:42 -175: f() at reporters/tests.R:42 DONE =========================================================================== diff --git a/tests/testthat/reporters/tap.txt b/tests/testthat/reporters/tap.txt index 1f35ac9e2..1b560f673 100644 --- a/tests/testthat/reporters/tap.txt +++ b/tests/testthat/reporters/tap.txt @@ -40,6 +40,8 @@ not ok 9 Recursion:1 9: f() at reporters/tests.R:42 10: f() at reporters/tests.R:42 ... + 164: f() at reporters/tests.R:42 + 165: f() at reporters/tests.R:42 166: f() at reporters/tests.R:42 167: f() at reporters/tests.R:42 168: f() at reporters/tests.R:42 @@ -48,8 +50,6 @@ not ok 9 Recursion:1 171: f() at reporters/tests.R:42 172: f() at reporters/tests.R:42 173: f() at reporters/tests.R:42 - 174: f() at reporters/tests.R:42 - 175: f() at reporters/tests.R:42 # Context Skips ok 10 # SKIP skip ok 11 # SKIP skip diff --git a/tests/testthat/reporters/teamcity.txt b/tests/testthat/reporters/teamcity.txt index b6341b4fe..bc03e249a 100644 --- a/tests/testthat/reporters/teamcity.txt +++ b/tests/testthat/reporters/teamcity.txt @@ -52,7 +52,7 @@ ##teamcity[testSuiteStarted name='Recursion'] ##teamcity[testSuiteStarted name='Recursion:1'] ##teamcity[testStarted name='expectation 1'] -##teamcity[testFailed name='expectation 1' message='evaluation nested too deeply: infinite recursion / options(expressions=)?' details='1: f() at reporters/tests.R:43|n2: f() at reporters/tests.R:42|n3: f() at reporters/tests.R:42|n4: f() at reporters/tests.R:42|n5: f() at reporters/tests.R:42|n6: f() at reporters/tests.R:42|n7: f() at reporters/tests.R:42|n8: f() at reporters/tests.R:42|n9: f() at reporters/tests.R:42|n10: f() at reporters/tests.R:42|n...|n166: f() at reporters/tests.R:42|n167: f() at reporters/tests.R:42|n168: f() at reporters/tests.R:42|n169: f() at reporters/tests.R:42|n170: f() at reporters/tests.R:42|n171: f() at reporters/tests.R:42|n172: f() at reporters/tests.R:42|n173: f() at reporters/tests.R:42|n174: f() at reporters/tests.R:42|n175: f() at reporters/tests.R:42'] +##teamcity[testFailed name='expectation 1' message='evaluation nested too deeply: infinite recursion / options(expressions=)?' details='1: f() at reporters/tests.R:43|n2: f() at reporters/tests.R:42|n3: f() at reporters/tests.R:42|n4: f() at reporters/tests.R:42|n5: f() at reporters/tests.R:42|n6: f() at reporters/tests.R:42|n7: f() at reporters/tests.R:42|n8: f() at reporters/tests.R:42|n9: f() at reporters/tests.R:42|n10: f() at reporters/tests.R:42|n...|n164: f() at reporters/tests.R:42|n165: f() at reporters/tests.R:42|n166: f() at reporters/tests.R:42|n167: f() at reporters/tests.R:42|n168: f() at reporters/tests.R:42|n169: f() at reporters/tests.R:42|n170: f() at reporters/tests.R:42|n171: f() at reporters/tests.R:42|n172: f() at reporters/tests.R:42|n173: f() at reporters/tests.R:42'] ##teamcity[testFinished name='expectation 1'] ##teamcity[testSuiteFinished name='Recursion:1'] diff --git a/tests/testthat/test-context.R b/tests/testthat/test-context.R index 2fda7b05d..0030a5675 100644 --- a/tests/testthat/test-context.R +++ b/tests/testthat/test-context.R @@ -28,7 +28,7 @@ CountReporter <- R6::R6Class("CountReporter", inherit = Reporter, test_that("contexts are opened, then closed", { report <- CountReporter$new() - test_file("context.R", report) + test_file("context.R", report, wrap = FALSE) expect_that(report$context_count, equals(2)) expect_that(report$context_i, equals(0)) expect_that(report$test_count, equals(4)) diff --git a/tests/testthat/test-encoding.R b/tests/testthat/test-encoding.R index 431459e02..ea4e87cca 100644 --- a/tests/testthat/test-encoding.R +++ b/tests/testthat/test-encoding.R @@ -1,17 +1,17 @@ context("encoding") test_that("can source file with Latin-1 encoding", { - expect_error(test_file("latin1.R", "stop", encoding = "latin1"), NA) + expect_error(test_file("latin1.R", "stop", encoding = "latin1", wrap = FALSE), NA) }) test_that("can source file with UTF-8 encoding", { - expect_error(test_file("utf8.R", "stop", encoding = "UTF-8"), NA) + expect_error(test_file("utf8.R", "stop", encoding = "UTF-8", wrap = FALSE), NA) }) test_that("error with Latin-1 encoding mismatch", { - expect_error(test_file("latin1.R", "stop", encoding = "UTF-8")) + expect_error(test_file("latin1.R", "stop", encoding = "UTF-8", wrap = FALSE)) }) test_that("error with UTF-8 encoding mismatch", { - expect_error(test_file("utf8.R", "stop", encoding = "latin1")) + expect_error(test_file("utf8.R", "stop", encoding = "latin1", wrap = FALSE)) }) diff --git a/tests/testthat/test-reporter.R b/tests/testthat/test-reporter.R index 4d73788b5..8460f16f1 100644 --- a/tests/testthat/test-reporter.R +++ b/tests/testthat/test-reporter.R @@ -49,7 +49,7 @@ test_that("reporters produce consistent output", { expect_error( withr::with_options( list(expressions = Cstack_info()[["eval_depth"]] + 200), - test_file(test_path("reporters/tests.R"), reporter) + test_file(test_path("reporters/tests.R"), reporter, wrap = FALSE) ), error_regexp ), diff --git a/tests/testthat/test-source_dir.R b/tests/testthat/test-source_dir.R index 68fde6184..f0449a36f 100644 --- a/tests/testthat/test-source_dir.R +++ b/tests/testthat/test-source_dir.R @@ -1,15 +1,15 @@ context("source_dir") test_that('source_dir()', { - res <- source_dir('test_dir', pattern = 'hello', chdir = TRUE) + res <- source_dir('test_dir', pattern = 'hello', chdir = TRUE, wrap = FALSE) expect_equal(res[[1]](), "Hello World") - - res <- source_dir(normalizePath('test_dir'), pattern = 'hello', chdir = TRUE) + + res <- source_dir(normalizePath('test_dir'), pattern = 'hello', chdir = TRUE, wrap = FALSE) + expect_equal(res[[1]](), "Hello World") + + res <- source_dir('test_dir', pattern = 'hello', chdir = FALSE, wrap = FALSE) expect_equal(res[[1]](), "Hello World") - res <- source_dir('test_dir', pattern = 'hello', chdir = FALSE) - expect_equal(res[[1]](), "Hello World") - - res <- source_dir(normalizePath('test_dir'), pattern = 'hello', chdir = FALSE) + res <- source_dir(normalizePath('test_dir'), pattern = 'hello', chdir = FALSE, wrap = FALSE) expect_equal(res[[1]](), "Hello World") })