From fe2ea0ec905eff275c76e8006382f7c101ac89cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Tue, 1 Mar 2016 11:17:21 +0100 Subject: [PATCH 1/2] Test error output in expect_message() with update of golden files to show the problem --- tests/testthat/reporters/check.txt | 7 ------- tests/testthat/reporters/minimal.txt | 2 +- tests/testthat/reporters/rstudio.txt | 1 - tests/testthat/reporters/summary.txt | 16 +++++++++++----- tests/testthat/reporters/tap.txt | 14 ++++++++++---- tests/testthat/reporters/teamcity.txt | 3 --- tests/testthat/reporters/tests.R | 2 +- 7 files changed, 23 insertions(+), 22 deletions(-) diff --git a/tests/testthat/reporters/check.txt b/tests/testthat/reporters/check.txt index c2dc2fda0..262c000a9 100644 --- a/tests/testthat/reporters/check.txt +++ b/tests/testthat/reporters/check.txt @@ -20,13 +20,6 @@ FALSE isn't true. stop 1: stop("stop") at reporters/tests.R:28 -6. Error: Error:3 (@tests.R#36) ------------------------------------------------ -! -1: f() at reporters/tests.R:36 -2: g() at reporters/tests.R:32 -3: h() at reporters/tests.R:33 -4: stop("!") at reporters/tests.R:34 - testthat results ================================================================ OK: 5 SKIPPED: 2 FAILED: 6 1. Failure: Failure:1 (@tests.R#8) diff --git a/tests/testthat/reporters/minimal.txt b/tests/testthat/reporters/minimal.txt index 7635dc98a..7d1381296 100644 --- a/tests/testthat/reporters/minimal.txt +++ b/tests/testthat/reporters/minimal.txt @@ -1 +1 @@ -.FFFF.EESSWWW +.FFFF.ESSWWW diff --git a/tests/testthat/reporters/rstudio.txt b/tests/testthat/reporters/rstudio.txt index 99c3907da..c4360df49 100644 --- a/tests/testthat/reporters/rstudio.txt +++ b/tests/testthat/reporters/rstudio.txt @@ -3,7 +3,6 @@ reporters/tests.R#12:1 [failure] Failure:2a. Failure has been forced reporters/tests.R#15:1 [failure] Failure:2b. FALSE isn't true. reporters/tests.R#20:1 [failure] Failure:loop. `i` not equal to 2. reporters/tests.R#28:1 [error] Error:1. stop -reporters/tests.R#36:1 [error] Error:3. ! reporters/tests.R#42:1 [skip] Skip:1. skip reporters/tests.R#47:1 [skip] Skip:2. skip reporters/tests.R#53:1 [warning] Warning:1. abc diff --git a/tests/testthat/reporters/summary.txt b/tests/testthat/reporters/summary.txt index 806da3b6c..1db77c69c 100644 --- a/tests/testthat/reporters/summary.txt +++ b/tests/testthat/reporters/summary.txt @@ -1,5 +1,5 @@ Expectations: .1234. -Errors: 56 +Errors: 5 Skips: SS Warnings: WWW @@ -40,9 +40,15 @@ stop 6. Error: Error:3 (@tests.R#36) ------------------------------------------------ ! -1: f() at reporters/tests.R:36 -2: g() at reporters/tests.R:32 -3: h() at reporters/tests.R:33 -4: stop("!") at reporters/tests.R:34 +1: expect_message(f(), NA) at reporters/tests.R:36 +2: evaluate_promise(object, capture_warnings = FALSE) at /home/muelleki/git/R/testthat/R/expect-output.R:145 +3: with_sink(temp, withCallingHandlers(withVisible(code), warning = handle_warning, + message = handle_message)) at /home/muelleki/git/R/testthat/R/evaluate-promise.R:56 +4: withCallingHandlers(withVisible(code), warning = handle_warning, message = handle_message) at /home/muelleki/git/R/testthat/R/evaluate-promise.R:87 +5: withVisible(code) +6: f() +7: g() at reporters/tests.R:32 +8: h() at reporters/tests.R:33 +9: stop("!") at reporters/tests.R:34 DONE ======================================================================= diff --git a/tests/testthat/reporters/tap.txt b/tests/testthat/reporters/tap.txt index f785642b5..226443f97 100644 --- a/tests/testthat/reporters/tap.txt +++ b/tests/testthat/reporters/tap.txt @@ -22,10 +22,16 @@ not ok 7 Error:1 1: stop("stop") at reporters/tests.R:28 not ok 8 Error:3 ! - 1: f() at reporters/tests.R:36 - 2: g() at reporters/tests.R:32 - 3: h() at reporters/tests.R:33 - 4: stop("!") at reporters/tests.R:34 + 1: expect_message(f(), NA) at reporters/tests.R:36 + 2: evaluate_promise(object, capture_warnings = FALSE) at /home/muelleki/git/R/testthat/R/expect-output.R:145 + 3: with_sink(temp, withCallingHandlers(withVisible(code), warning = handle_warning, + message = handle_message)) at /home/muelleki/git/R/testthat/R/evaluate-promise.R:56 + 4: withCallingHandlers(withVisible(code), warning = handle_warning, message = handle_message) at /home/muelleki/git/R/testthat/R/evaluate-promise.R:87 + 5: withVisible(code) + 6: f() + 7: g() at reporters/tests.R:32 + 8: h() at reporters/tests.R:33 + 9: stop("!") at reporters/tests.R:34 # Context Skips ok 9 # SKIP skip ok 10 # SKIP skip diff --git a/tests/testthat/reporters/teamcity.txt b/tests/testthat/reporters/teamcity.txt index 31d3dea6e..5e856989d 100644 --- a/tests/testthat/reporters/teamcity.txt +++ b/tests/testthat/reporters/teamcity.txt @@ -41,9 +41,6 @@ ##teamcity[testSuiteFinished name='Error:1'] ##teamcity[testSuiteStarted name='Error:3'] -##teamcity[testStarted name='expectation 1'] -##teamcity[testFailed name='expectation 1' message='!' details='1: f() at reporters/tests.R:36|n2: g() at reporters/tests.R:32|n3: h() at reporters/tests.R:33|n4: stop("!") at reporters/tests.R:34'] -##teamcity[testFinished name='expectation 1'] ##teamcity[testSuiteFinished name='Error:3'] ##teamcity[testSuiteFinished name='Errors'] diff --git a/tests/testthat/reporters/tests.R b/tests/testthat/reporters/tests.R index 17b25f91c..521df7b99 100644 --- a/tests/testthat/reporters/tests.R +++ b/tests/testthat/reporters/tests.R @@ -33,7 +33,7 @@ test_that("Error:3", { g <- function() {h()} h <- function() {stop("!")} - f() + expect_message(f(), NA) }) context("Skips") From 51df2e8ac650889418957c2cfa9d54dc362e0741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Fri, 4 Mar 2016 17:12:42 +0100 Subject: [PATCH 2/2] demonstrate actual failure --- tests/testthat/reporters/check.txt | 2 +- tests/testthat/reporters/minimal.txt | 2 +- tests/testthat/reporters/summary.txt | 6 ++++++ tests/testthat/reporters/tap.txt | 5 ++++- tests/testthat/reporters/teamcity.txt | 10 ++++++++++ tests/testthat/reporters/tests.R | 6 ++++++ 6 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/testthat/reporters/check.txt b/tests/testthat/reporters/check.txt index 262c000a9..a000f69e1 100644 --- a/tests/testthat/reporters/check.txt +++ b/tests/testthat/reporters/check.txt @@ -21,7 +21,7 @@ stop 1: stop("stop") at reporters/tests.R:28 testthat results ================================================================ -OK: 5 SKIPPED: 2 FAILED: 6 +OK: 7 SKIPPED: 2 FAILED: 6 1. Failure: Failure:1 (@tests.R#8) 2. Failure: Failure:2a (@tests.R#12) 3. Failure: Failure:2b (@tests.R#15) diff --git a/tests/testthat/reporters/minimal.txt b/tests/testthat/reporters/minimal.txt index 7d1381296..7613be39f 100644 --- a/tests/testthat/reporters/minimal.txt +++ b/tests/testthat/reporters/minimal.txt @@ -1 +1 @@ -.FFFF.ESSWWW +.FFFF.ESSWWWF diff --git a/tests/testthat/reporters/summary.txt b/tests/testthat/reporters/summary.txt index 1db77c69c..53debb9c6 100644 --- a/tests/testthat/reporters/summary.txt +++ b/tests/testthat/reporters/summary.txt @@ -2,6 +2,7 @@ Expectations: .1234. Errors: 5 Skips: SS Warnings: WWW +Promises: 7 Skipped -------------------------------------------------------------------- 1. Skip:1 (@tests.R#42) - skip @@ -51,4 +52,9 @@ stop 8: h() at reporters/tests.R:33 9: stop("!") at reporters/tests.R:34 +7. Failure: Promises:1 (@tests.R#64) ------------------------------------------- +expect_false(FALSE) produced output. +. + + DONE ======================================================================= diff --git a/tests/testthat/reporters/tap.txt b/tests/testthat/reporters/tap.txt index 226443f97..e2ff10f02 100644 --- a/tests/testthat/reporters/tap.txt +++ b/tests/testthat/reporters/tap.txt @@ -1,4 +1,4 @@ -1..13 +1..15 # Context Expectations ok 1 Success not ok 2 Failure:1 @@ -39,3 +39,6 @@ ok 10 # SKIP skip ok 11 # WARNING abc ok 12 # WARNING def ok 13 # WARNING ghi +# Context Promises +ok 14 Promises:1 +ok 15 Promises:1 diff --git a/tests/testthat/reporters/teamcity.txt b/tests/testthat/reporters/teamcity.txt index 5e856989d..626de678e 100644 --- a/tests/testthat/reporters/teamcity.txt +++ b/tests/testthat/reporters/teamcity.txt @@ -74,3 +74,13 @@ ##teamcity[testSuiteFinished name='Warnings'] +##teamcity[testSuiteStarted name='Promises'] +##teamcity[testSuiteStarted name='Promises:1'] +##teamcity[testStarted name='expectation 2'] +##teamcity[testFailed name='expectation 2' message='expect_false(FALSE) produced output.' details='##teamcity|[testStarted name=|'expectation 1|'|]\n##teamcity|[testFinished name=|'expectation 1|'|]'] +##teamcity[testFinished name='expectation 2'] +##teamcity[testSuiteFinished name='Promises:1'] + +##teamcity[testSuiteFinished name='Promises'] + + diff --git a/tests/testthat/reporters/tests.R b/tests/testthat/reporters/tests.R index 521df7b99..22f742f79 100644 --- a/tests/testthat/reporters/tests.R +++ b/tests/testthat/reporters/tests.R @@ -57,3 +57,9 @@ test_that("Warning:2", { warning("def") warning("ghi") }) + +context("Promises") + +test_that("Promises:1", { + expect_output(expect_false(FALSE), NA) +})