New option testthat.summary.omit_dots to omit progress dots (#502)

* simplify

* omit dots if option testthat.summary.omit_dots is set

* extract method

* add test for omit_dots = TRUE

* NEWS

* edge case

* abort testing as soon as limit is reached

* line break

* describe option

NEWS.md

@@ -1,5 +1,13 @@
 # testthat 1.0.2.9000
 
+* The default summary reporter aborts testing as soon as the limit given by the
+  option `testthat.summary.max_reports` (default 15) is reached
+  (#520).
+
+* New option `testthat.summary.omit_dots`, default to `FALSE`. Setting to `TRUE`
+  hides the progress dots from the output of the summary reporter, which speeds
+  up tests by a small margin (#502).
+
 * New `LocationReporter` which just prints the location of every expectation.
   This is useful for locating segfaults and C/C++ breakpoints (#551).
 

R/reporter-summary.R

@@ -22,42 +22,49 @@ SummaryReporter <- R6::R6Class("SummaryReporter", inherit = Reporter,
     failures = NULL,
     skips = NULL,
     warnings = NULL,
-    max_reports = getOption("testthat.summary.max_reports", 15L),
+    max_reports = NULL,
     show_praise = TRUE,
+    omit_dots = FALSE,
 
-    initialize = function(show_praise = TRUE) {
+    initialize = function(show_praise = TRUE, omit_dots = getOption("testthat.summary.omit_dots"), max_reports = getOption("testthat.summary.max_reports", 15L)) {
       super$initialize()
-      self$show_praise <- show_praise
       self$failures <- Stack$new()
       self$skips <- Stack$new()
       self$warnings <- Stack$new()
+      self$max_reports <- max_reports
+      self$show_praise <- show_praise
+      self$omit_dots <- omit_dots
     },
 
     start_context = function(context) {
       self$cat_tight(context, ": ")
     },
 
+    end_test = function(context, test) {
+      if (self$failures$size() >= self$max_reports) {
+        self$cat_line()
+        stop("Reached maximum number of reports.", call. = FALSE)
+      }
+    },
+
     end_context = function(context) {
       self$cat_line()
     },
 
     add_result = function(context, test, result) {
       if (expectation_broken(result)) {
-        if (self$failures$size() + 1 > self$max_reports) {
-          self$cat_tight(single_letter_summary(result))
-        } else {
-          self$failures$push(result)
-          self$cat_tight(colourise(labels[self$failures$size()], "error"))
-        }
+        self$failures$push(result)
       } else if (expectation_skip(result)) {
         self$skips$push(result)
-        self$cat_tight(single_letter_summary(result))
       } else if (expectation_warning(result)) {
         self$warnings$push(result)
-        self$cat_tight(single_letter_summary(result))
       } else {
-        self$cat_tight(single_letter_summary(result))
+        if (isTRUE(self$omit_dots)) {
+          return()
+        }
       }
+
+      self$cat_tight(private$get_summary(result))
     },
 
     end_reporter = function() {
@@ -83,6 +90,16 @@ SummaryReporter <- R6::R6Class("SummaryReporter", inherit = Reporter,
   ),
 
   private = list(
+    get_summary = function(result) {
+      if (expectation_broken(result)) {
+        if (self$failures$size() <= length(labels)) {
+          return(colourise(labels[self$failures$size()], "error"))
+        }
+      }
+
+      single_letter_summary(result)
+    },
+
     cat_reports = function(header, expectations, max_n, summary_fun,
                            collapse = "\n\n") {
       n <- length(expectations)

R/test-that.R

@@ -178,12 +178,15 @@ test_code <- function(test, code, env = test_env(), skip_on_empty = TRUE) {
 #' and integrates with your existing workflow.
 #'
 #' @section Options:
-#' `testthat.use_colours`: Should the output be coloured? (Default:
+#' - `testthat.use_colours`: Should the output be coloured? (Default:
 #' `TRUE`).
 #'
-#' `testthat.summary.max_reports`: The maximum number of detailed test
+#' - `testthat.summary.max_reports`: The maximum number of detailed test
 #' reports printed for the summary reporter (default: 15).
 #'
+#' - `testthat.summary.omit_dots`: Omit progress dots in the summary reporter
+#' (default: `FALSE`).
+#'
 #' @docType package
 #' @name testthat
 #' @references Wickham, H (2011). testthat: Get Started with Testing.

tests/testthat/reporters/summary-2.txt

@@ -0,0 +1,12 @@
+Expectations: .12
+
+Failed -------------------------------------------------------------------------
+1. Failure: Failure:1 (@tests.R#8) ---------------------------------------------
+Failure has been forced
+
+
+2. Failure: Failure:2a (@tests.R#12) -------------------------------------------
+Failure has been forced
+
+
+DONE ===========================================================================

tests/testthat/reporters/summary-no-dots.txt

@@ -0,0 +1,76 @@
+Expectations: 1234
+Errors: 56
+Recursion: 7
+Skips: SSS
+Warnings: WWW
+Output: 
+
+Skipped ------------------------------------------------------------------------
+1. Skip:1 (@tests.R#49) - skip
+
+2. Skip:2 (@tests.R#54) - skip
+
+3. Skip:3 (@tests.R#57) - Empty test
+
+Warnings -----------------------------------------------------------------------
+1. Warning:1 (@tests.R#63) - abc
+
+2. Warning:2 (@tests.R#67) - def
+
+3. Warning:2 (@tests.R#68) - ghi
+
+Failed -------------------------------------------------------------------------
+1. Failure: Failure:1 (@tests.R#8) ---------------------------------------------
+Failure has been forced
+
+
+2. Failure: Failure:2a (@tests.R#12) -------------------------------------------
+Failure has been forced
+
+
+3. Failure: Failure:2b (@tests.R#15) -------------------------------------------
+FALSE isn't true.
+
+
+4. Failure: Failure:loop (@tests.R#20) -----------------------------------------
+`i` not equal to 2.
+1/1 mismatches
+[1] 1 - 2 == -1
+
+
+5. Error: Error:1 (@tests.R#28) ------------------------------------------------
+stop
+1: stop("stop") at reporters/tests.R:28
+
+6. Error: Error:3 (@tests.R#36) ------------------------------------------------
+!
+1: f() at reporters/tests.R:36
+2: g() at reporters/tests.R:32
+3: h() at reporters/tests.R:33
+4: stop("!") at reporters/tests.R:34
+
+7. Error: Recursion:1 (@tests.R#43) --------------------------------------------
+evaluation nested too deeply: infinite recursion / options(expressions=)?
+1: f() at reporters/tests.R:43
+2: f() at reporters/tests.R:42
+3: f() at reporters/tests.R:42
+4: f() at reporters/tests.R:42
+5: f() at reporters/tests.R:42
+6: f() at reporters/tests.R:42
+7: f() at reporters/tests.R:42
+8: f() at reporters/tests.R:42
+9: f() at reporters/tests.R:42
+10: f() at reporters/tests.R:42
+...
+166: f() at reporters/tests.R:42
+167: f() at reporters/tests.R:42
+168: f() at reporters/tests.R:42
+169: f() at reporters/tests.R:42
+170: f() at reporters/tests.R:42
+171: f() at reporters/tests.R:42
+172: f() at reporters/tests.R:42
+173: f() at reporters/tests.R:42
+174: f() at reporters/tests.R:42
+175: f() at reporters/tests.R:42
+
+DONE ===========================================================================

tests/testthat/test-reporter.R

@@ -65,7 +65,10 @@ test_that("reporters produce consistent output", {
     save_report("debug")
   )
   save_report("check", error_regexp = NULL)
-  save_report("summary", SummaryReporter$new(show_praise = FALSE))
+  save_report("summary", SummaryReporter$new(show_praise = FALSE, omit_dots = FALSE))
+  save_report("summary-2", SummaryReporter$new(show_praise = FALSE, max_reports = 2),
+              error_regexp = "Reached maximum number of reports")
+  save_report("summary-no-dots", SummaryReporter$new(show_praise = FALSE, omit_dots = TRUE))
   save_report("location")
   save_report("minimal")
   save_report("tap")