Skip to content

Commit

Permalink
Fix tests for older Gradle versions
Browse files Browse the repository at this point in the history 
Signed-off-by: Pavlo Shevchenko <[email protected]>
  • Loading branch information
@pshevche
pshevche committed Jul 12, 2024
1 parent 4416228 commit f2ce44e
Show file tree
Hide file tree
Showing 3 changed files with 117 additions and 73 deletions.
101 changes: 46 additions & 55 deletions plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,13 @@ import spock.lang.Issue
import javax.annotation.Nullable
import java.util.regex.Pattern

import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_CLASS
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_METHOD
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_TEST
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_CLASS
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_METHOD
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_TEST

abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
@Override
String getLanguagePlugin() {
Expand All @@ -36,13 +43,30 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
"""
}

abstract String reportedLifecycleMethodName(String methodName)
enum TestNGLifecycleType {
BEFORE_SUITE('BeforeSuite'),
BEFORE_TEST('BeforeTest'),
BEFORE_CLASS('BeforeClass'),
BEFORE_METHOD('BeforeMethod'),
AFTER_METHOD('AfterMethod'),
AFTER_CLASS('AfterClass'),
AFTER_TEST('AfterTest'),
AFTER_SUITE('AfterSuite')

final String annotation

TestNGLifecycleType(String annotation) {
this.annotation = annotation
}
}

abstract String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation)
abstract String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName)

abstract boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType)
abstract String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation)

def "handles failure in #lifecycle (gradle version #gradleVersion)"() {
abstract boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType)

def "handles failure in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) {
given:
buildFile << """
test.retry.maxRetries = 1
Expand All @@ -52,8 +76,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
package acme;
public class SuccessfulTests {
@org.testng.annotations.${lifecycle}
public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() {
@org.testng.annotations.${lifecycle.annotation}
public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() {
${flakyAssert()}
}
Expand All @@ -67,20 +91,20 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {

then:
with(result.output) {
it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 1
it.count("${reportedLifecycleMethodName('lifecycle')} PASSED") == (reportsSuccessfulLifecycleExecutions(lifecycle) ? 1 : 0)
it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} FAILED") == 1
it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} PASSED") == (reportsSuccessfulLifecycleExecutions(lifecycle) ? 1 : 0)
!it.contains("The following test methods could not be retried")
}

where:
[gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [
GRADLE_VERSIONS_UNDER_TEST,
['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest']
[BEFORE_TEST, BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD, AFTER_CLASS, AFTER_TEST]
])
// Note: we don't handle BeforeSuite AfterSuite
}

def "correctly reports exhausted retries on failures in #lifecycle (gradle version #gradleVersion)"() {
def "correctly reports exhausted retries on failures in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) {
given:
buildFile << """
test.retry.maxRetries = 1
Expand All @@ -90,8 +114,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
package acme;
public class AlwaysFailingLifecycle {
@org.testng.annotations.${lifecycle}
public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() {
@org.testng.annotations.${lifecycle.annotation}
public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() {
throw new RuntimeException("Lifecycle goes boom!");
}
Expand All @@ -106,52 +130,19 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
then:
with(result.output) {
// if BeforeTest fails, then methods won't be executed
it.count('successTest SKIPPED') == (lifecycle.contains('Before') ? 2 : 0)
it.count('successTest PASSED') == (lifecycle.contains('Before') ? 0 : 2)
it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 2
it.count('successTest SKIPPED') == (lifecycle.annotation.contains('Before') ? 2 : 0)
it.count('successTest PASSED') == (lifecycle.annotation.contains('Before') ? 0 : 2)
it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} FAILED") == 2
!it.contains("The following test methods could not be retried")
}

where:
[gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [
GRADLE_VERSIONS_UNDER_TEST,
['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest']
[BEFORE_TEST, BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD, AFTER_CLASS, AFTER_TEST]
])
}

def "does not handle flaky static initializers (gradle version #gradleVersion)"() {
given:
buildFile << """
test.retry.maxRetries = 1
"""
writeJavaTestSource """
package acme;

public class SomeTests {

static {
${flakyAssert()}
}

@org.testng.annotations.Test
public void someTest() {}
}
"""
when:
def result = gradleRunner(gradleVersion as String).buildAndFail()
then:
with(result.output) {
it.contains('There were failing tests. See the report')
!it.contains('The following test methods could not be retried')
}
where:
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
}
def "handles parameterized test in super class (gradle version #gradleVersion)"() {
given:
buildFile << """
Expand Down Expand Up @@ -191,8 +182,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
then:
// we can't rerun just the failed parameter
with(result.output) {
it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2
it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2
it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 0, '0')} PASSED") == 2
it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 1, '1')} FAILED") == 2
}
where:
Expand Down Expand Up @@ -316,8 +307,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
then:
// we can't rerun just the failed parameter
with(result.output) {
it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2
it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2
it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 0, '0')} PASSED") == 2
it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 1, '1')} FAILED") == 2
}
where:
Expand Down Expand Up @@ -369,8 +360,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
then:
// we can't rerun just the failed parameter
with(result.output.readLines()) {
it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 0, ''))}.* PASSED/) }.size() == 2
it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 1, ''))}.* FAILED/) }.size() == 2
it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName(gradleVersion, 'test', 'acme.ParameterTest$Foo', 0, ''))}.* PASSED/) }.size() == 2
it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName(gradleVersion, 'test', 'acme.ParameterTest$Foo', 1, ''))}.* FAILED/) }.size() == 2
}
where:
Expand Down
42 changes: 39 additions & 3 deletions plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,53 @@ import javax.annotation.Nullable

class TestNGPlainFuncTest extends BaseTestNGFuncTest {
@Override
String reportedLifecycleMethodName(String methodName) {
String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) {
methodName
}

@Override
String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) {
String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) {
"${methodName}[${invocationNumber}]${paramValueRepresentation ? "(${paramValueRepresentation})" : ""}"
}

@Override
boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) {
boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) {
true
}

/**
* If JUnit's TestNG engine is used, then tests won't even run and the failure is silently swallowed.
*/
def "does not handle flaky static initializers (gradle version #gradleVersion)"() {
given:
buildFile << """
test.retry.maxRetries = 1
"""
writeJavaTestSource """
package acme;

public class SomeTests {

static {
${flakyAssert()}
}

@org.testng.annotations.Test
public void someTest() {}
}
"""
when:
def result = gradleRunner(gradleVersion as String).buildAndFail()
then:
with(result.output) {
it.contains('There were failing tests. See the report')
!it.contains('The following test methods could not be retried')
}
where:
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
}
}
47 changes: 32 additions & 15 deletions ...in/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,24 @@
*/
package org.gradle.testretry.testframework

import org.gradle.util.GradleVersion

import javax.annotation.Nullable

import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_CLASS
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_METHOD
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_TEST
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_CLASS
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_METHOD
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_TEST

class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest {

private static final Set<String> UNREPORTED_LIFECYCLE_METHODS = ['BeforeTest', 'AfterTest', 'AfterClass']
private static final EnumSet<TestNGLifecycleType> UNREPORTED_LIFECYCLE_METHODS = EnumSet.of(BEFORE_TEST, AFTER_TEST, AFTER_CLASS)
private static final EnumSet<TestNGLifecycleType> CLASS_LIFECYCLE_METHODS = EnumSet.of(BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD)

private static final GradleVersion GRADLE_5_0 = GradleVersion.version("5.0")
private static final GradleVersion GRADLE_5_4_1 = GradleVersion.version("5.4.1")

def setup() {
buildFile << """
Expand All @@ -34,21 +47,25 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest {
}

@Override
String reportedLifecycleMethodName(String methodName) {
"executionError"
String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) {
GradleVersion.version(gradleVersion) > GRADLE_5_0
? "executionError"
: CLASS_LIFECYCLE_METHODS.contains(lifecycleType) ? "classMethod" : "initializationError"
}

@Override
String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) {
"${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}"
String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) {
GradleVersion.version(gradleVersion) > GRADLE_5_4_1
? "${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}"
: "${methodName}(${paramType})[${invocationNumber}]"
}

@Override
boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) {
!UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleMethodType)
boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) {
!UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleType)
}

def "retries all classes if failure occurs in #lifecycle (gradle version #gradleVersion)"() {
def "retries all classes if failure occurs in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) {
given:
buildFile << """
test.retry.maxRetries = 1
Expand All @@ -58,8 +75,8 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest {
package acme;
public class SuccessfulTestsWithFailingLifecycle {
@org.testng.annotations.${lifecycle}
public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() {
@org.testng.annotations.${lifecycle.annotation}
public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() {
${flakyAssert()}
}
Expand All @@ -83,18 +100,18 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest {
then:
with(result.output) {
// if BeforeTest fails, then methods won't be executed
it.count('successTest SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0)
it.count('successTestWithLifecycle SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0)
it.count('successTest SKIPPED') == (lifecycle == BEFORE_TEST ? 1 : 0)
it.count('successTestWithLifecycle SKIPPED') == (lifecycle == BEFORE_TEST ? 1 : 0)

it.count('successTest PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2)
it.count('successTestWithLifecycle PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2)
it.count('successTest PASSED') == (lifecycle == BEFORE_TEST ? 1 : 2)
it.count('successTestWithLifecycle PASSED') == (lifecycle == BEFORE_TEST ? 1 : 2)
!it.contains("The following test methods could not be retried")
}

where:
[gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [
GRADLE_VERSIONS_UNDER_TEST,
['BeforeTest', 'AfterClass', 'AfterTest']
UNREPORTED_LIFECYCLE_METHODS
])
}
}

0 comments on commit f2ce44e

Please sign in to comment.