From 2f2726e917ef29e8707e6b5292cf26b93f50ecee Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Thu, 11 Jul 2024 11:53:27 +0200 Subject: [PATCH 1/6] Add test coverage for retrying TestNG tests run with JUnit TestNG engine Signed-off-by: Pavlo Shevchenko --- ...cTest.groovy => BaseTestNGFuncTest.groovy} | 25 +++++++---- .../testframework/TestNGPlainFuncTest.groovy | 30 +++++++++++++ .../TestNGViaJUnitEngineFuncTest.groovy | 43 +++++++++++++++++++ 3 files changed, 89 insertions(+), 9 deletions(-) rename plugin/src/test/groovy/org/gradle/testretry/testframework/{TestNGFuncTest.groovy => BaseTestNGFuncTest.groovy} (92%) create mode 100644 plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy create mode 100644 plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy similarity index 92% rename from plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGFuncTest.groovy rename to plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy index 9b1badf1..b953a1e5 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy @@ -18,7 +18,10 @@ package org.gradle.testretry.testframework import org.gradle.testretry.AbstractFrameworkFuncTest import spock.lang.Issue -class TestNGFuncTest extends AbstractFrameworkFuncTest { +import javax.annotation.Nullable +import java.util.regex.Pattern + +abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { @Override String getLanguagePlugin() { return 'java' @@ -33,6 +36,10 @@ class TestNGFuncTest extends AbstractFrameworkFuncTest { """ } + abstract String reportedLifecycleMethodName(String methodName) + + abstract String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) + def "handles failure in #lifecycle (gradle version #gradleVersion)"() { given: buildFile << """ @@ -58,8 +65,8 @@ class TestNGFuncTest extends AbstractFrameworkFuncTest { then: with(result.output) { - it.count('lifecycle FAILED') == 1 - it.count('lifecycle PASSED') == 1 + it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 1 + it.count("${reportedLifecycleMethodName('lifecycle')} PASSED") == 1 !it.contains("The following test methods could not be retried") } @@ -143,8 +150,8 @@ class TestNGFuncTest extends AbstractFrameworkFuncTest { then: // we can't rerun just the failed parameter with(result.output) { - it.count('test[0](0) PASSED') == 2 - it.count('test[1](1) FAILED') == 2 + it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2 + it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2 } where: @@ -266,8 +273,8 @@ class TestNGFuncTest extends AbstractFrameworkFuncTest { then: // we can't rerun just the failed parameter with(result.output) { - it.count('test[0](0) PASSED') == 2 - it.count('test[1](1) FAILED') == 2 + it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2 + it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2 } where: @@ -319,8 +326,8 @@ class TestNGFuncTest extends AbstractFrameworkFuncTest { then: // we can't rerun just the failed parameter with(result.output.readLines()) { - it.findAll { line -> line.matches('.*test\\[0].* PASSED') }.size() == 2 - it.findAll { line -> line.matches('.*test\\[1].* FAILED') }.size() == 2 + it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 0, ''))}.* PASSED/) }.size() == 2 + it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 1, ''))}.* FAILED/) }.size() == 2 } where: diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy new file mode 100644 index 00000000..ab30833c --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy @@ -0,0 +1,30 @@ +/* + * Copyright 2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.testframework + +import javax.annotation.Nullable + +class TestNGPlainFuncTest extends BaseTestNGFuncTest { + @Override + String reportedLifecycleMethodName(String methodName) { + methodName + } + + @Override + String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { + "${methodName}[${invocationNumber}]${paramValueRepresentation ? "(${paramValueRepresentation})" : ""}" + } +} diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy new file mode 100644 index 00000000..fc6d30d7 --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy @@ -0,0 +1,43 @@ +/* + * Copyright 2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.testframework + +import javax.annotation.Nullable + +class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { + + def setup() { + buildFile << """ + dependencies { + testImplementation 'org.testng:testng:7.5' + testRuntimeOnly 'org.junit.support:testng-engine:1.0.5' + } + test { + useJUnitPlatform() + } + """ + } + + @Override + String reportedLifecycleMethodName(String methodName) { + "executionError" + } + + @Override + String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { + "${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}" + } +} From dbf7ea680dee67bc9901215eee4d7623c65ec177 Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Thu, 11 Jul 2024 14:11:55 +0200 Subject: [PATCH 2/6] Correctly retry TestNG tests with dependencies when using JUnit engine Signed-off-by: Pavlo Shevchenko --- .../BaseJunitTestFrameworkStrategy.java | 26 ++++++++++++++++++- .../testframework/BaseTestNGFuncTest.groovy | 10 ++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java index 83e63871..b3e13a72 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java @@ -77,6 +77,10 @@ protected void addFilters(TestFilterBuilder filters, TestsReader testsReader, Te return; } + if (processTestNGTest(filters, testsReader, className, tests)) { + return; + } + tests.forEach(name -> addPotentiallyParameterizedSuffixed(filters, className, name)); }); } @@ -113,11 +117,31 @@ private boolean processSpockTest(TestFilterBuilder filters, TestsReader testsRea return true; } } catch (Throwable t) { - LOGGER.warn("Unable to determine if class " + className + " contains Spock @Unroll parameterizations", t); + LOGGER.warn("Unable to determine if class {} contains Spock @Unroll parameterizations", className, t); } return false; } + private boolean processTestNGTest(TestFilterBuilder filters, TestsReader testsReader, String className, Set tests) { + try { + Optional resultOpt = testsReader.readTestClassDirClass(className, TestNgClassVisitor::new); + if (resultOpt.isPresent()) { + TestNgClassVisitor.ClassInfo result = resultOpt.get(); + + tests.forEach(test -> { + addPotentiallyParameterizedSuffixed(filters, className, test); + result.dependsOn(test).forEach(dependency -> filters.test(className, dependency)); + }); + + return true; + } + } catch (Throwable t) { + LOGGER.warn("Unable to determine dependency between methods of a TestNG test class {}", className, t); + } + + return false; + } + private void addPotentiallyParameterizedSuffixed(TestFilterBuilder filters, String className, String name) { // It's a common pattern to add all the parameters on the end of a literal method name with [] // The regex takes care of removing trailing (...) or (...)[...], for e.g. the following cases diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy index b953a1e5..c9919756 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy @@ -212,7 +212,7 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { public class OrderedTests { @Test(dependsOnMethods = {"childTest"}) - public void grandchildTest() {} + public void grandChildTest() {} @Test(dependsOnMethods = {"parentTest"}) public void childTest() { @@ -229,12 +229,14 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: with(result.output) { - it.count('childTest FAILED') == 1 it.count('parentTest PASSED') == 2 + it.count('childTest FAILED') == 1 + it.count('childTest PASSED') == 1 + // grandchildTest gets skipped initially because flaky childTest failed, but is ran as part of the retry - it.count('grandchildTest SKIPPED') == 1 - it.count('grandchildTest PASSED') == 1 + it.count('grandChildTest SKIPPED') == 1 + it.count('grandChildTest PASSED') == 1 } where: From fff4a658f90db5358d44bd4939cc7a55f99b0c20 Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Thu, 11 Jul 2024 15:40:08 +0200 Subject: [PATCH 3/6] Retry TestNG lifecycle methods not attributed to a test class Signed-off-by: Pavlo Shevchenko --- .../internal/executer/RetryTestExecuter.java | 2 +- .../executer/RetryTestResultProcessor.java | 25 ++++++++++++++++--- .../internal/executer/RoundResult.java | 7 +++++- .../BaseJunitTestFrameworkStrategy.java | 25 +++++++++++++++---- .../Junit5TestFrameworkStrategy.java | 5 ++-- .../framework/JunitTestFrameworkStrategy.java | 5 ++-- .../framework/TestFrameworkStrategy.java | 6 ++++- .../TestNgTestFrameworkStrategy.java | 2 +- .../testframework/BaseTestNGFuncTest.groovy | 6 +++-- .../testframework/TestNGPlainFuncTest.groovy | 5 ++++ .../TestNGViaJUnitEngineFuncTest.groovy | 7 ++++++ 11 files changed, 77 insertions(+), 18 deletions(-) diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestExecuter.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestExecuter.java index 92c5025f..06a70655 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestExecuter.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestExecuter.java @@ -129,7 +129,7 @@ public void execute(JvmTestExecutionSpec spec, TestResultProcessor testResultPro } else if (result.lastRound) { break; } else { - TestFramework retryTestFramework = testFrameworkStrategy.createRetrying(frameworkTemplate, spec.getTestFramework(), result.failedTests); + TestFramework retryTestFramework = testFrameworkStrategy.createRetrying(frameworkTemplate, spec.getTestFramework(), result.failedTests, result.testClassesSeenInCurrentRound); testExecutionSpec = testExecutionSpecFor(retryTestFramework, spec); retryTestResultProcessor.reset(++retryCount == maxRetries); } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestResultProcessor.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestResultProcessor.java index db0912ee..f7102862 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestResultProcessor.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestResultProcessor.java @@ -33,7 +33,9 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; +import static java.util.stream.Collectors.toMap; import static org.gradle.api.tasks.testing.TestResult.ResultType.SKIPPED; final class RetryTestResultProcessor implements TestResultProcessor { @@ -91,6 +93,10 @@ public void started(TestDescriptorInternal descriptor, TestStartEvent testStartE @Override public void completed(Object testId, TestCompleteEvent testCompleteEvent) { if (testId.equals(rootTestDescriptorId)) { + // nothing failed in the current round, but we have some un-retried tests + if (currentRoundFailedTests.isEmpty() && !previousRoundFailedTests.isEmpty()) { + ignoreExpectedUnretriedTests(); + } if (!lastRun()) { return; } @@ -127,13 +133,24 @@ public void completed(Object testId, TestCompleteEvent testCompleteEvent) { } }); } - } } delegate.completed(testId, testCompleteEvent); } + private void ignoreExpectedUnretriedTests() { + // check with the framework implementation if it is expected + Map> expectedUnretriedTests = previousRoundFailedTests.stream() + .collect(toMap( + Map.Entry::getKey, + entry -> entry.getValue().stream() + .filter(test -> testFrameworkStrategy.isExpectedUnretriedTest(entry.getKey(), test)) + .collect(Collectors.toSet()) + )); + expectedUnretriedTests.forEach((className, tests) -> previousRoundFailedTests.remove(className, tests::contains)); + } + private boolean isLifecycleFailure(String className, String name) { return testFrameworkStrategy.isLifecycleFailureTest(testsReader, className, name); } @@ -253,10 +270,12 @@ private boolean currentRoundFailedTestsExceedsMaxFailures() { } public RoundResult getResult() { - return new RoundResult(currentRoundFailedTests, + return new RoundResult( + currentRoundFailedTests, cleanedUpFailedTestsOfPreviousRound(), lastRun(), - hasRetryFilteredFailures + hasRetryFilteredFailures, + testClassesSeenInCurrentRound ); } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/RoundResult.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/RoundResult.java index 388e1aa0..e94332b5 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/RoundResult.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/RoundResult.java @@ -15,22 +15,27 @@ */ package org.gradle.testretry.internal.executer; +import java.util.Set; + final class RoundResult { final TestNames failedTests; final TestNames nonRetriedTests; final boolean lastRound; final boolean hasRetryFilteredFailures; + final Set testClassesSeenInCurrentRound; RoundResult( TestNames failedTests, TestNames nonRetriedTests, boolean lastRound, - boolean hasRetryFilteredFailures + boolean hasRetryFilteredFailures, + Set testClassesSeenInCurrentRound ) { this.failedTests = failedTests; this.nonRetriedTests = nonRetriedTests; this.lastRound = lastRound; this.hasRetryFilteredFailures = hasRetryFilteredFailures; + this.testClassesSeenInCurrentRound = testClassesSeenInCurrentRound; } } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java index b3e13a72..e5fe9de4 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java @@ -36,6 +36,7 @@ abstract class BaseJunitTestFrameworkStrategy implements TestFrameworkStrategy { public static final Logger LOGGER = LoggerFactory.getLogger(JunitTestFrameworkStrategy.class); private static final Pattern PARAMETERIZED_SUFFIX_PATTERN = Pattern.compile("(?:\\([^)]*?\\))?(?:\\[[^]]*?])?$"); + private static final String ERROR_SYNTHETIC_TESTNG_CLASS_NAME = "UnknownClass"; static final Set ERROR_SYNTHETIC_TEST_NAMES = Collections.unmodifiableSet( new HashSet<>(Arrays.asList( "classMethod", @@ -50,14 +51,18 @@ public boolean isLifecycleFailureTest(TestsReader testsReader, String className, return ERROR_SYNTHETIC_TEST_NAMES.contains(testName); } - protected DefaultTestFilter testFilterFor(TestNames failedTests, boolean canRunParameterizedSpockMethods, TestFrameworkTemplate template) { - TestFilterBuilder filter = template.filterBuilder(); - addFilters(filter, template.testsReader, failedTests, canRunParameterizedSpockMethods); + @Override + public boolean isExpectedUnretriedTest(String className, String test) { + return ERROR_SYNTHETIC_TESTNG_CLASS_NAME.equals(className); + } + protected DefaultTestFilter testFilterFor(TestNames failedTests, boolean canRunParameterizedSpockMethods, TestFrameworkTemplate template, Set testClassesSeenInCurrentRound) { + TestFilterBuilder filter = template.filterBuilder(); + addFilters(filter, template.testsReader, failedTests, canRunParameterizedSpockMethods, testClassesSeenInCurrentRound); return filter.build(); } - protected void addFilters(TestFilterBuilder filters, TestsReader testsReader, TestNames failedTests, boolean canRunParameterizedSpockMethods) { + protected void addFilters(TestFilterBuilder filters, TestsReader testsReader, TestNames failedTests, boolean canRunParameterizedSpockMethods, Set testClassesSeenInCurrentRound) { failedTests.stream() .forEach(entry -> { String className = entry.getKey(); @@ -69,7 +74,17 @@ protected void addFilters(TestFilterBuilder filters, TestsReader testsReader, Te } if (tests.stream().anyMatch(ERROR_SYNTHETIC_TEST_NAMES::contains)) { - filters.clazz(className); + if (ERROR_SYNTHETIC_TESTNG_CLASS_NAME.equals(className)) { + // Gradle can't properly attribute some of the TestNG lifecycle method failures to classes + // Those are @BeforeTest, @AfterTest, @AfterClass methods + // In case of @BeforeTest and @AfterTest, we should retry all classes belonging to the TestNG Test + // However, we don't know this association and retry all classes instead + // In case of @AfterClass, we should be able to retry just a single class + // Due to erroneous reporting from TestNG we don't know which one it is + testClassesSeenInCurrentRound.forEach(filters::clazz); + } else { + filters.clazz(className); + } return; } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/Junit5TestFrameworkStrategy.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/Junit5TestFrameworkStrategy.java index c34de030..87bd8f2c 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/Junit5TestFrameworkStrategy.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/Junit5TestFrameworkStrategy.java @@ -24,6 +24,7 @@ import org.gradle.testretry.internal.executer.framework.TestFrameworkProvider.ProviderForCurrentGradleVersion; import java.lang.reflect.Constructor; +import java.util.Set; import static org.gradle.testretry.internal.executer.framework.Junit5TestFrameworkStrategy.Junit5TestFrameworkProvider.testFrameworkProvider; import static org.gradle.testretry.internal.executer.framework.TestFrameworkStrategy.gradleVersionIsAtLeast; @@ -37,8 +38,8 @@ public Junit5TestFrameworkStrategy(boolean isSpock2Used) { } @Override - public TestFramework createRetrying(TestFrameworkTemplate template, TestFramework testFramework, TestNames failedTests) { - DefaultTestFilter failedTestsFilter = testFilterFor(failedTests, isSpock2Used, template); + public TestFramework createRetrying(TestFrameworkTemplate template, TestFramework testFramework, TestNames failedTests, Set testClassesSeenInCurrentRound) { + DefaultTestFilter failedTestsFilter = testFilterFor(failedTests, isSpock2Used, template, testClassesSeenInCurrentRound); return testFrameworkProvider(template, testFramework).testFrameworkFor(failedTestsFilter); } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/JunitTestFrameworkStrategy.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/JunitTestFrameworkStrategy.java index 781d2118..d02a4453 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/JunitTestFrameworkStrategy.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/JunitTestFrameworkStrategy.java @@ -24,6 +24,7 @@ import org.gradle.testretry.internal.executer.TestNames; import java.lang.reflect.Constructor; +import java.util.Set; import static org.gradle.testretry.internal.executer.framework.JunitTestFrameworkStrategy.JunitTestFrameworkProvider.testFrameworkProvider; import static org.gradle.testretry.internal.executer.framework.TestFrameworkStrategy.gradleVersionIsAtLeast; @@ -31,8 +32,8 @@ final class JunitTestFrameworkStrategy extends BaseJunitTestFrameworkStrategy implements TestFrameworkStrategy { @Override - public TestFramework createRetrying(TestFrameworkTemplate template, TestFramework testFramework, TestNames failedTests) { - DefaultTestFilter failedTestsFilter = testFilterFor(failedTests, true, template); + public TestFramework createRetrying(TestFrameworkTemplate template, TestFramework testFramework, TestNames failedTests, Set testClassesSeenInCurrentRound) { + DefaultTestFilter failedTestsFilter = testFilterFor(failedTests, true, template, testClassesSeenInCurrentRound); return testFrameworkProvider(template, testFramework).testFrameworkFor(failedTestsFilter); } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestFrameworkStrategy.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestFrameworkStrategy.java index 9ea69464..48d117a0 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestFrameworkStrategy.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestFrameworkStrategy.java @@ -27,6 +27,7 @@ import javax.annotation.Nullable; import java.io.File; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.StreamSupport; @@ -74,6 +75,9 @@ static boolean gradleVersionIsAtLeast(String version) { boolean isLifecycleFailureTest(TestsReader testsReader, String className, String testName); - TestFramework createRetrying(TestFrameworkTemplate template, TestFramework testFramework, TestNames failedTests); + TestFramework createRetrying(TestFrameworkTemplate template, TestFramework testFramework, TestNames failedTests, Set testClassesSeenInCurrentRound); + default boolean isExpectedUnretriedTest(String className, String test) { + return false; + } } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgTestFrameworkStrategy.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgTestFrameworkStrategy.java index 0659ecfe..abe2735c 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgTestFrameworkStrategy.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgTestFrameworkStrategy.java @@ -70,7 +70,7 @@ private boolean isLifecycleMethod(TestsReader testsReader, String testName, Test } @Override - public TestFramework createRetrying(TestFrameworkTemplate template, TestFramework testFramework, TestNames failedTests) { + public TestFramework createRetrying(TestFrameworkTemplate template, TestFramework testFramework, TestNames failedTests, Set testClassesSeenInCurrentRound) { DefaultTestFilter failedTestsFilter = testFilterFor(failedTests, template); return testFrameworkProvider(template, testFramework) diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy index c9919756..f103f16f 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy @@ -40,6 +40,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { abstract String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) + abstract boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) + def "handles failure in #lifecycle (gradle version #gradleVersion)"() { given: buildFile << """ @@ -66,14 +68,14 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: with(result.output) { it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 1 - it.count("${reportedLifecycleMethodName('lifecycle')} PASSED") == 1 + it.count("${reportedLifecycleMethodName('lifecycle')} PASSED") == (reportsSuccessfulLifecycleExecutions(lifecycle) ? 1 : 0) !it.contains("The following test methods could not be retried") } where: [gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [ GRADLE_VERSIONS_UNDER_TEST, - ['BeforeClass', 'BeforeTest', 'BeforeMethod', 'AfterClass', 'AfterTest', 'AfterMethod'] + ['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest'] ]) // Note: we don't handle BeforeSuite AfterSuite } diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy index ab30833c..62a90c4a 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy @@ -27,4 +27,9 @@ class TestNGPlainFuncTest extends BaseTestNGFuncTest { String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { "${methodName}[${invocationNumber}]${paramValueRepresentation ? "(${paramValueRepresentation})" : ""}" } + + @Override + boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) { + true + } } diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy index fc6d30d7..1e05ad2b 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy @@ -19,6 +19,8 @@ import javax.annotation.Nullable class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { + private static final Set UNREPORTED_LIFECYCLE_METHODS = ['BeforeTest', 'AfterTest', 'AfterClass'] + def setup() { buildFile << """ dependencies { @@ -40,4 +42,9 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { "${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}" } + + @Override + boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) { + !UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleMethodType) + } } From 433527ebf6546ade73990443001fd2454014e5b0 Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Thu, 11 Jul 2024 15:46:35 +0200 Subject: [PATCH 4/6] Add more tests for retrying TestNG lifecycle methods Signed-off-by: Pavlo Shevchenko --- .../testframework/BaseTestNGFuncTest.groovy | 39 +++++++++++++++ .../TestNGViaJUnitEngineFuncTest.groovy | 50 +++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy index f103f16f..ee2a7b70 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy @@ -80,6 +80,45 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { // Note: we don't handle BeforeSuite AfterSuite } + def "correctly reports exhausted retries on failures in #lifecycle (gradle version #gradleVersion)"() { + given: + buildFile << """ + test.retry.maxRetries = 1 + """ + + writeJavaTestSource """ + package acme; + + public class AlwaysFailingLifecycle { + @org.testng.annotations.${lifecycle} + public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() { + throw new RuntimeException("Lifecycle goes boom!"); + } + + @org.testng.annotations.Test + public void successTest() {} + } + """ + + when: + def result = gradleRunner(gradleVersion as String).buildAndFail() + + then: + with(result.output) { + // if BeforeTest fails, then methods won't be executed + it.count('successTest SKIPPED') == (lifecycle.contains('Before') ? 2 : 0) + it.count('successTest PASSED') == (lifecycle.contains('Before') ? 0 : 2) + it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 2 + !it.contains("The following test methods could not be retried") + } + + where: + [gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [ + GRADLE_VERSIONS_UNDER_TEST, + ['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest'] + ]) + } + def "does not handle flaky static initializers (gradle version #gradleVersion)"() { given: buildFile << """ diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy index 1e05ad2b..0fbe1c15 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy @@ -47,4 +47,54 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) { !UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleMethodType) } + + def "retries all classes if failure occurs in #lifecycle (gradle version #gradleVersion)"() { + given: + buildFile << """ + test.retry.maxRetries = 1 + """ + + writeJavaTestSource """ + package acme; + + public class SuccessfulTestsWithFailingLifecycle { + @org.testng.annotations.${lifecycle} + public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() { + ${flakyAssert()} + } + + @org.testng.annotations.Test + public void successTestWithLifecycle() {} + } + """ + + writeJavaTestSource """ + package acme; + + public class SuccessfulTestsPotentiallyDependingOnLifecycle { + @org.testng.annotations.Test + public void successTest() {} + } + """ + + when: + def result = gradleRunner(gradleVersion as String).build() + + then: + with(result.output) { + // if BeforeTest fails, then methods won't be executed + it.count('successTest SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0) + it.count('successTestWithLifecycle SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0) + + it.count('successTest PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2) + it.count('successTestWithLifecycle PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2) + !it.contains("The following test methods could not be retried") + } + + where: + [gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [ + GRADLE_VERSIONS_UNDER_TEST, + ['BeforeTest', 'AfterClass', 'AfterTest'] + ]) + } } From 44162288f14cb915a50dcbbb8898120f5ff255ae Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Thu, 11 Jul 2024 16:06:39 +0200 Subject: [PATCH 5/6] Fix license header Signed-off-by: Pavlo Shevchenko --- .../gradle/testretry/testframework/TestNGPlainFuncTest.groovy | 2 +- .../testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy index 62a90c4a..cef50d6b 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2024 the original author or authors. + * Copyright 2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy index 0fbe1c15..a569f248 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2024 the original author or authors. + * Copyright 2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From f2ce44e2dfd4ef2b4a717998fc0f22cb201fd402 Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Fri, 12 Jul 2024 09:02:30 +0200 Subject: [PATCH 6/6] Fix tests for older Gradle versions Signed-off-by: Pavlo Shevchenko --- .../testframework/BaseTestNGFuncTest.groovy | 101 ++++++++---------- .../testframework/TestNGPlainFuncTest.groovy | 42 +++++++- .../TestNGViaJUnitEngineFuncTest.groovy | 47 +++++--- 3 files changed, 117 insertions(+), 73 deletions(-) diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy index ee2a7b70..499f1921 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy @@ -21,6 +21,13 @@ import spock.lang.Issue import javax.annotation.Nullable import java.util.regex.Pattern +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_CLASS +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_METHOD +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_TEST +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_CLASS +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_METHOD +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_TEST + abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { @Override String getLanguagePlugin() { @@ -36,13 +43,30 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { """ } - abstract String reportedLifecycleMethodName(String methodName) + enum TestNGLifecycleType { + BEFORE_SUITE('BeforeSuite'), + BEFORE_TEST('BeforeTest'), + BEFORE_CLASS('BeforeClass'), + BEFORE_METHOD('BeforeMethod'), + AFTER_METHOD('AfterMethod'), + AFTER_CLASS('AfterClass'), + AFTER_TEST('AfterTest'), + AFTER_SUITE('AfterSuite') + + final String annotation + + TestNGLifecycleType(String annotation) { + this.annotation = annotation + } + } - abstract String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) + abstract String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) - abstract boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) + abstract String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) - def "handles failure in #lifecycle (gradle version #gradleVersion)"() { + abstract boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) + + def "handles failure in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) { given: buildFile << """ test.retry.maxRetries = 1 @@ -52,8 +76,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { package acme; public class SuccessfulTests { - @org.testng.annotations.${lifecycle} - public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() { + @org.testng.annotations.${lifecycle.annotation} + public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() { ${flakyAssert()} } @@ -67,20 +91,20 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: with(result.output) { - it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 1 - it.count("${reportedLifecycleMethodName('lifecycle')} PASSED") == (reportsSuccessfulLifecycleExecutions(lifecycle) ? 1 : 0) + it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} FAILED") == 1 + it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} PASSED") == (reportsSuccessfulLifecycleExecutions(lifecycle) ? 1 : 0) !it.contains("The following test methods could not be retried") } where: [gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [ GRADLE_VERSIONS_UNDER_TEST, - ['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest'] + [BEFORE_TEST, BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD, AFTER_CLASS, AFTER_TEST] ]) // Note: we don't handle BeforeSuite AfterSuite } - def "correctly reports exhausted retries on failures in #lifecycle (gradle version #gradleVersion)"() { + def "correctly reports exhausted retries on failures in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) { given: buildFile << """ test.retry.maxRetries = 1 @@ -90,8 +114,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { package acme; public class AlwaysFailingLifecycle { - @org.testng.annotations.${lifecycle} - public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() { + @org.testng.annotations.${lifecycle.annotation} + public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() { throw new RuntimeException("Lifecycle goes boom!"); } @@ -106,52 +130,19 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: with(result.output) { // if BeforeTest fails, then methods won't be executed - it.count('successTest SKIPPED') == (lifecycle.contains('Before') ? 2 : 0) - it.count('successTest PASSED') == (lifecycle.contains('Before') ? 0 : 2) - it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 2 + it.count('successTest SKIPPED') == (lifecycle.annotation.contains('Before') ? 2 : 0) + it.count('successTest PASSED') == (lifecycle.annotation.contains('Before') ? 0 : 2) + it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} FAILED") == 2 !it.contains("The following test methods could not be retried") } where: [gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [ GRADLE_VERSIONS_UNDER_TEST, - ['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest'] + [BEFORE_TEST, BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD, AFTER_CLASS, AFTER_TEST] ]) } - def "does not handle flaky static initializers (gradle version #gradleVersion)"() { - given: - buildFile << """ - test.retry.maxRetries = 1 - """ - - writeJavaTestSource """ - package acme; - - public class SomeTests { - - static { - ${flakyAssert()} - } - - @org.testng.annotations.Test - public void someTest() {} - } - """ - - when: - def result = gradleRunner(gradleVersion as String).buildAndFail() - - then: - with(result.output) { - it.contains('There were failing tests. See the report') - !it.contains('The following test methods could not be retried') - } - - where: - gradleVersion << GRADLE_VERSIONS_UNDER_TEST - } - def "handles parameterized test in super class (gradle version #gradleVersion)"() { given: buildFile << """ @@ -191,8 +182,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: // we can't rerun just the failed parameter with(result.output) { - it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2 - it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2 + it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 0, '0')} PASSED") == 2 + it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 1, '1')} FAILED") == 2 } where: @@ -316,8 +307,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: // we can't rerun just the failed parameter with(result.output) { - it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2 - it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2 + it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 0, '0')} PASSED") == 2 + it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 1, '1')} FAILED") == 2 } where: @@ -369,8 +360,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: // we can't rerun just the failed parameter with(result.output.readLines()) { - it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 0, ''))}.* PASSED/) }.size() == 2 - it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 1, ''))}.* FAILED/) }.size() == 2 + it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName(gradleVersion, 'test', 'acme.ParameterTest$Foo', 0, ''))}.* PASSED/) }.size() == 2 + it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName(gradleVersion, 'test', 'acme.ParameterTest$Foo', 1, ''))}.* FAILED/) }.size() == 2 } where: diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy index cef50d6b..258d891d 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy @@ -19,17 +19,53 @@ import javax.annotation.Nullable class TestNGPlainFuncTest extends BaseTestNGFuncTest { @Override - String reportedLifecycleMethodName(String methodName) { + String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) { methodName } @Override - String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { + String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { "${methodName}[${invocationNumber}]${paramValueRepresentation ? "(${paramValueRepresentation})" : ""}" } @Override - boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) { + boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) { true } + + /** + * If JUnit's TestNG engine is used, then tests won't even run and the failure is silently swallowed. + */ + def "does not handle flaky static initializers (gradle version #gradleVersion)"() { + given: + buildFile << """ + test.retry.maxRetries = 1 + """ + + writeJavaTestSource """ + package acme; + + public class SomeTests { + + static { + ${flakyAssert()} + } + + @org.testng.annotations.Test + public void someTest() {} + } + """ + + when: + def result = gradleRunner(gradleVersion as String).buildAndFail() + + then: + with(result.output) { + it.contains('There were failing tests. See the report') + !it.contains('The following test methods could not be retried') + } + + where: + gradleVersion << GRADLE_VERSIONS_UNDER_TEST + } } diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy index a569f248..41e637b2 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy @@ -15,11 +15,24 @@ */ package org.gradle.testretry.testframework +import org.gradle.util.GradleVersion + import javax.annotation.Nullable +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_CLASS +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_METHOD +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_TEST +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_CLASS +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_METHOD +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_TEST + class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { - private static final Set UNREPORTED_LIFECYCLE_METHODS = ['BeforeTest', 'AfterTest', 'AfterClass'] + private static final EnumSet UNREPORTED_LIFECYCLE_METHODS = EnumSet.of(BEFORE_TEST, AFTER_TEST, AFTER_CLASS) + private static final EnumSet CLASS_LIFECYCLE_METHODS = EnumSet.of(BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD) + + private static final GradleVersion GRADLE_5_0 = GradleVersion.version("5.0") + private static final GradleVersion GRADLE_5_4_1 = GradleVersion.version("5.4.1") def setup() { buildFile << """ @@ -34,21 +47,25 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { } @Override - String reportedLifecycleMethodName(String methodName) { - "executionError" + String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) { + GradleVersion.version(gradleVersion) > GRADLE_5_0 + ? "executionError" + : CLASS_LIFECYCLE_METHODS.contains(lifecycleType) ? "classMethod" : "initializationError" } @Override - String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { - "${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}" + String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { + GradleVersion.version(gradleVersion) > GRADLE_5_4_1 + ? "${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}" + : "${methodName}(${paramType})[${invocationNumber}]" } @Override - boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) { - !UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleMethodType) + boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) { + !UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleType) } - def "retries all classes if failure occurs in #lifecycle (gradle version #gradleVersion)"() { + def "retries all classes if failure occurs in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) { given: buildFile << """ test.retry.maxRetries = 1 @@ -58,8 +75,8 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { package acme; public class SuccessfulTestsWithFailingLifecycle { - @org.testng.annotations.${lifecycle} - public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() { + @org.testng.annotations.${lifecycle.annotation} + public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() { ${flakyAssert()} } @@ -83,18 +100,18 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { then: with(result.output) { // if BeforeTest fails, then methods won't be executed - it.count('successTest SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0) - it.count('successTestWithLifecycle SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0) + it.count('successTest SKIPPED') == (lifecycle == BEFORE_TEST ? 1 : 0) + it.count('successTestWithLifecycle SKIPPED') == (lifecycle == BEFORE_TEST ? 1 : 0) - it.count('successTest PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2) - it.count('successTestWithLifecycle PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2) + it.count('successTest PASSED') == (lifecycle == BEFORE_TEST ? 1 : 2) + it.count('successTestWithLifecycle PASSED') == (lifecycle == BEFORE_TEST ? 1 : 2) !it.contains("The following test methods could not be retried") } where: [gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [ GRADLE_VERSIONS_UNDER_TEST, - ['BeforeTest', 'AfterClass', 'AfterTest'] + UNREPORTED_LIFECYCLE_METHODS ]) } }