From f2ce44e2dfd4ef2b4a717998fc0f22cb201fd402 Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Fri, 12 Jul 2024 09:02:30 +0200 Subject: [PATCH] Fix tests for older Gradle versions Signed-off-by: Pavlo Shevchenko --- .../testframework/BaseTestNGFuncTest.groovy | 101 ++++++++---------- .../testframework/TestNGPlainFuncTest.groovy | 42 +++++++- .../TestNGViaJUnitEngineFuncTest.groovy | 47 +++++--- 3 files changed, 117 insertions(+), 73 deletions(-) diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy index ee2a7b70..499f1921 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy @@ -21,6 +21,13 @@ import spock.lang.Issue import javax.annotation.Nullable import java.util.regex.Pattern +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_CLASS +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_METHOD +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_TEST +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_CLASS +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_METHOD +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_TEST + abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { @Override String getLanguagePlugin() { @@ -36,13 +43,30 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { """ } - abstract String reportedLifecycleMethodName(String methodName) + enum TestNGLifecycleType { + BEFORE_SUITE('BeforeSuite'), + BEFORE_TEST('BeforeTest'), + BEFORE_CLASS('BeforeClass'), + BEFORE_METHOD('BeforeMethod'), + AFTER_METHOD('AfterMethod'), + AFTER_CLASS('AfterClass'), + AFTER_TEST('AfterTest'), + AFTER_SUITE('AfterSuite') + + final String annotation + + TestNGLifecycleType(String annotation) { + this.annotation = annotation + } + } - abstract String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) + abstract String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) - abstract boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) + abstract String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) - def "handles failure in #lifecycle (gradle version #gradleVersion)"() { + abstract boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) + + def "handles failure in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) { given: buildFile << """ test.retry.maxRetries = 1 @@ -52,8 +76,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { package acme; public class SuccessfulTests { - @org.testng.annotations.${lifecycle} - public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() { + @org.testng.annotations.${lifecycle.annotation} + public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() { ${flakyAssert()} } @@ -67,20 +91,20 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: with(result.output) { - it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 1 - it.count("${reportedLifecycleMethodName('lifecycle')} PASSED") == (reportsSuccessfulLifecycleExecutions(lifecycle) ? 1 : 0) + it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} FAILED") == 1 + it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} PASSED") == (reportsSuccessfulLifecycleExecutions(lifecycle) ? 1 : 0) !it.contains("The following test methods could not be retried") } where: [gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [ GRADLE_VERSIONS_UNDER_TEST, - ['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest'] + [BEFORE_TEST, BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD, AFTER_CLASS, AFTER_TEST] ]) // Note: we don't handle BeforeSuite AfterSuite } - def "correctly reports exhausted retries on failures in #lifecycle (gradle version #gradleVersion)"() { + def "correctly reports exhausted retries on failures in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) { given: buildFile << """ test.retry.maxRetries = 1 @@ -90,8 +114,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { package acme; public class AlwaysFailingLifecycle { - @org.testng.annotations.${lifecycle} - public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() { + @org.testng.annotations.${lifecycle.annotation} + public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() { throw new RuntimeException("Lifecycle goes boom!"); } @@ -106,52 +130,19 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: with(result.output) { // if BeforeTest fails, then methods won't be executed - it.count('successTest SKIPPED') == (lifecycle.contains('Before') ? 2 : 0) - it.count('successTest PASSED') == (lifecycle.contains('Before') ? 0 : 2) - it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 2 + it.count('successTest SKIPPED') == (lifecycle.annotation.contains('Before') ? 2 : 0) + it.count('successTest PASSED') == (lifecycle.annotation.contains('Before') ? 0 : 2) + it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} FAILED") == 2 !it.contains("The following test methods could not be retried") } where: [gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [ GRADLE_VERSIONS_UNDER_TEST, - ['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest'] + [BEFORE_TEST, BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD, AFTER_CLASS, AFTER_TEST] ]) } - def "does not handle flaky static initializers (gradle version #gradleVersion)"() { - given: - buildFile << """ - test.retry.maxRetries = 1 - """ - - writeJavaTestSource """ - package acme; - - public class SomeTests { - - static { - ${flakyAssert()} - } - - @org.testng.annotations.Test - public void someTest() {} - } - """ - - when: - def result = gradleRunner(gradleVersion as String).buildAndFail() - - then: - with(result.output) { - it.contains('There were failing tests. See the report') - !it.contains('The following test methods could not be retried') - } - - where: - gradleVersion << GRADLE_VERSIONS_UNDER_TEST - } - def "handles parameterized test in super class (gradle version #gradleVersion)"() { given: buildFile << """ @@ -191,8 +182,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: // we can't rerun just the failed parameter with(result.output) { - it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2 - it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2 + it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 0, '0')} PASSED") == 2 + it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 1, '1')} FAILED") == 2 } where: @@ -316,8 +307,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: // we can't rerun just the failed parameter with(result.output) { - it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2 - it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2 + it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 0, '0')} PASSED") == 2 + it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 1, '1')} FAILED") == 2 } where: @@ -369,8 +360,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest { then: // we can't rerun just the failed parameter with(result.output.readLines()) { - it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 0, ''))}.* PASSED/) }.size() == 2 - it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 1, ''))}.* FAILED/) }.size() == 2 + it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName(gradleVersion, 'test', 'acme.ParameterTest$Foo', 0, ''))}.* PASSED/) }.size() == 2 + it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName(gradleVersion, 'test', 'acme.ParameterTest$Foo', 1, ''))}.* FAILED/) }.size() == 2 } where: diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy index cef50d6b..258d891d 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy @@ -19,17 +19,53 @@ import javax.annotation.Nullable class TestNGPlainFuncTest extends BaseTestNGFuncTest { @Override - String reportedLifecycleMethodName(String methodName) { + String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) { methodName } @Override - String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { + String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { "${methodName}[${invocationNumber}]${paramValueRepresentation ? "(${paramValueRepresentation})" : ""}" } @Override - boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) { + boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) { true } + + /** + * If JUnit's TestNG engine is used, then tests won't even run and the failure is silently swallowed. + */ + def "does not handle flaky static initializers (gradle version #gradleVersion)"() { + given: + buildFile << """ + test.retry.maxRetries = 1 + """ + + writeJavaTestSource """ + package acme; + + public class SomeTests { + + static { + ${flakyAssert()} + } + + @org.testng.annotations.Test + public void someTest() {} + } + """ + + when: + def result = gradleRunner(gradleVersion as String).buildAndFail() + + then: + with(result.output) { + it.contains('There were failing tests. See the report') + !it.contains('The following test methods could not be retried') + } + + where: + gradleVersion << GRADLE_VERSIONS_UNDER_TEST + } } diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy index a569f248..41e637b2 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy @@ -15,11 +15,24 @@ */ package org.gradle.testretry.testframework +import org.gradle.util.GradleVersion + import javax.annotation.Nullable +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_CLASS +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_METHOD +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_TEST +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_CLASS +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_METHOD +import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_TEST + class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { - private static final Set UNREPORTED_LIFECYCLE_METHODS = ['BeforeTest', 'AfterTest', 'AfterClass'] + private static final EnumSet UNREPORTED_LIFECYCLE_METHODS = EnumSet.of(BEFORE_TEST, AFTER_TEST, AFTER_CLASS) + private static final EnumSet CLASS_LIFECYCLE_METHODS = EnumSet.of(BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD) + + private static final GradleVersion GRADLE_5_0 = GradleVersion.version("5.0") + private static final GradleVersion GRADLE_5_4_1 = GradleVersion.version("5.4.1") def setup() { buildFile << """ @@ -34,21 +47,25 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { } @Override - String reportedLifecycleMethodName(String methodName) { - "executionError" + String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) { + GradleVersion.version(gradleVersion) > GRADLE_5_0 + ? "executionError" + : CLASS_LIFECYCLE_METHODS.contains(lifecycleType) ? "classMethod" : "initializationError" } @Override - String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { - "${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}" + String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) { + GradleVersion.version(gradleVersion) > GRADLE_5_4_1 + ? "${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}" + : "${methodName}(${paramType})[${invocationNumber}]" } @Override - boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) { - !UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleMethodType) + boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) { + !UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleType) } - def "retries all classes if failure occurs in #lifecycle (gradle version #gradleVersion)"() { + def "retries all classes if failure occurs in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) { given: buildFile << """ test.retry.maxRetries = 1 @@ -58,8 +75,8 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { package acme; public class SuccessfulTestsWithFailingLifecycle { - @org.testng.annotations.${lifecycle} - public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() { + @org.testng.annotations.${lifecycle.annotation} + public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() { ${flakyAssert()} } @@ -83,18 +100,18 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest { then: with(result.output) { // if BeforeTest fails, then methods won't be executed - it.count('successTest SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0) - it.count('successTestWithLifecycle SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0) + it.count('successTest SKIPPED') == (lifecycle == BEFORE_TEST ? 1 : 0) + it.count('successTestWithLifecycle SKIPPED') == (lifecycle == BEFORE_TEST ? 1 : 0) - it.count('successTest PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2) - it.count('successTestWithLifecycle PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2) + it.count('successTest PASSED') == (lifecycle == BEFORE_TEST ? 1 : 2) + it.count('successTestWithLifecycle PASSED') == (lifecycle == BEFORE_TEST ? 1 : 2) !it.contains("The following test methods could not be retried") } where: [gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [ GRADLE_VERSIONS_UNDER_TEST, - ['BeforeTest', 'AfterClass', 'AfterTest'] + UNREPORTED_LIFECYCLE_METHODS ]) } }