Skip to content

Commit

Permalink
Bugfix on dealing with non-executed test phase due collateral issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@imonteroperez
imonteroperez committed Sep 15, 2021
1 parent 162dffc commit d442f46
Show file tree
Hide file tree
Showing 5 changed files with 67 additions and 22 deletions.
13 changes: 8 additions & 5 deletions plugins-compat-tester/src/main/java/org/jenkins/tools/test/PluginCompatTester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
Expand Down Expand Up @@ -310,7 +311,7 @@ && new VersionNumber(coreCoordinates.version).compareTo(new VersionNumber("1.485
if (errorMessage == null) {
try {
TestExecutionResult result = testPluginAgainst(actualCoreCoordinates, plugin, mconfig, pomData, pluginsToCheck, pluginGroupIds, pcth, config.getOverridenPlugins());
if (result.getTestDetails().getFailed().isEmpty()) {
if (result.getTestDetails().isSuccess()) {
status = TestStatus.SUCCESS;
} else {
status = TestStatus.TEST_FAILURES;
Expand All @@ -320,14 +321,16 @@ && new VersionNumber(coreCoordinates.version).compareTo(new VersionNumber("1.485
} catch (PomExecutionException e) {
if(!e.succeededPluginArtifactIds.contains("maven-compiler-plugin")){
status = TestStatus.COMPILATION_ERROR;
} else if (!e.getTestDetails().getFailed().isEmpty()) {
status = TestStatus.TEST_FAILURES;
} else if (!e.getTestDetails().hasBeenExecuted()) { // testing was not able to start properly (i.e: invalid exclusion list file format)
status = TestStatus.INTERNAL_ERROR;
} else if (!e.getTestDetails().hasFailures()) {
status = TestStatus.TEST_FAILURES;
} else { // Can this really happen ???
status = TestStatus.SUCCESS;
}
errorMessage = e.getErrorMessage();
warningMessages.addAll(e.getPomWarningMessages());
testDetails.addAll(config.isStoreAll() ? e.getTestDetails().getAll() : e.getTestDetails().getFailed());
testDetails.addAll(config.isStoreAll() ? e.getTestDetails().getAll() : e.getTestDetails().hasFailures() ? e.getTestDetails().getFailed() : Collections.emptySet());
} catch (Error e){
// Rethrow the error ... something is wrong !
throw e;
Expand Down Expand Up @@ -566,7 +569,7 @@ private TestExecutionResult testPluginAgainst(MavenCoordinates coreCoordinates,
} catch (PomExecutionException e){
e.getPomWarningMessages().addAll(pomData.getWarningMessages());
if (ranCompile) {
// So the status is considered to be TEST_FAILURES not COMPILATION_ERROR:
// So the status cannot be considered COMPILATION_ERROR
e.succeededPluginArtifactIds.add("maven-compiler-plugin");
}
throw e;
Expand Down
29 changes: 22 additions & 7 deletions ...ins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesDetails.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ public class ExecutedTestNamesDetails {

public ExecutedTestNamesDetails() {
this.tests = new HashMap<>();
this.tests.put(FAILED, new TreeSet<String>());
this.tests.put(EXECUTED, new TreeSet<String>());
}

public void addFailedTest(String test) {
Expand All @@ -29,8 +27,13 @@ public void addExecutedTest(String test) {
}

public Set<String> getAll() {
Set<String> result = new TreeSet<>(this.tests.get(EXECUTED));
result.addAll(this.tests.get(FAILED));
Set<String> result = new TreeSet<>();
if (this.tests.containsKey(EXECUTED)) {
result.addAll(this.tests.get(EXECUTED));
}
if (this.tests.containsKey(FAILED)) {
result.addAll(this.tests.get(FAILED));
}
return Collections.unmodifiableSet(result);
}

Expand All @@ -43,13 +46,25 @@ public Set<String> getExecuted() {
}

private Set<String> get(String key) {
return Collections.unmodifiableSet(new TreeSet<>(this.tests.get(key)));
return this.tests.containsKey(key) ? Collections.unmodifiableSet(new TreeSet<>(this.tests.get(key))) : null;
}

private void add(String key, String test) {
if (this.tests.get(key) == null) {
this.tests.put(key, new TreeSet<String>());
}
this.tests.get(key).add(test);
}



public boolean hasBeenExecuted() {
return getExecuted() != null || getFailed() != null;
}

public boolean isSuccess() {
return getExecuted() != null && getFailed() == null;
}

public boolean hasFailures() {
return getFailed() != null && !getFailed().isEmpty();
}
}
22 changes: 12 additions & 10 deletions plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesSolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,18 +101,20 @@ public ExecutedTestNamesDetails solve(Set<String> types, Set<String> executedTes

System.out.println("[INFO] ");
System.out.println("[INFO] Results:");
printDetails(testNames.getExecuted(), "Executed");
printDetails(testNames.getFailed(), "Failed");
return testNames;
}

private void printDetails(Set<String> tests, String type) {
System.out.println("[INFO] ");
System.out.println(String.format("[INFO] Executed: %s", testNames.getExecuted().size()));
for (String testName : testNames.getExecuted()) {
System.out.println(String.format("[INFO] - %s", testName));
}
System.out.println("[INFO] ");
System.out.println(String.format("[INFO] Failed: %s", testNames.getFailed().size()));
for (String testName : testNames.getFailed()) {
System.out.println(String.format("[INFO] - %s", testName));
int size = tests != null ? tests.size() : 0;
System.out.println(String.format("[INFO] %s: %s", type, size));
if (size != 0) {
for (String testName : tests) {
System.out.println(String.format("[INFO] - %s", testName));
}
}

return testNames;
}

private List<String> getReportsDirectoryPaths(Set<String> types, File baseDirectory) throws ExecutedTestNamesSolverException {
Expand Down
24 changes: 24 additions & 0 deletions plugins-compat-tester/src/test/java/org/jenkins/tools/test/PluginCompatTesterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,30 @@ public void testWithCasCProperties() throws Throwable {
PluginCompatTester tester = new PluginCompatTester(config);
tester.testPlugins();
}

@Test
public void testWithInvalidExclusionList() throws Throwable {
File exclusionList = new ClassPathResource("bad-surefire-exclusion-list").getFile();
PluginCompatTesterConfig config = getConfig(ImmutableList.of("active-directory"));
Map<String, String> mavenProperties = new HashMap<>();
mavenProperties.put("surefire.excludesFile", exclusionList.getAbsolutePath());
config.setMavenProperties(mavenProperties);

PluginCompatTester tester = new PluginCompatTester(config);
PluginCompatReport report = tester.testPlugins();
assertNotNull(report);
Map<PluginInfos, List<PluginCompatResult>> pluginCompatTests = report.getPluginCompatTests();
assertNotNull(pluginCompatTests);
for (Entry<PluginInfos, List<PluginCompatResult>> entry : pluginCompatTests.entrySet()) {
assertEquals("active-directory", entry.getKey().pluginName);
List<PluginCompatResult> results = entry.getValue();
assertEquals(1, results.size());
PluginCompatResult result = results.get(0);
assertNotNull(result);
assertNotNull(result.status);
assertEquals(TestStatus.INTERNAL_ERROR, result.status);
}
}

private PluginCompatTesterConfig getConfig(List<String> includedPlugins) throws IOException {
PluginCompatTesterConfig config = new PluginCompatTesterConfig(testFolder.getRoot(),
Expand Down
1 change: 1 addition & 0 deletions plugins-compat-tester/src/test/resources/bad-surefire-exclusion-list
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
io.jenkins.plugins.FooBar#method

0 comments on commit d442f46

Please sign in to comment.