Feature/generate report on interruption

core/src/main/java/eu/tsystems/mms/tic/testframework/common/JVMExitHook.java

@@ -0,0 +1,54 @@
+/*
+ * Testerra
+ *
+ * (C) 2024, Clemens Große, Deutsche Telekom MMS GmbH, Deutsche Telekom AG
+ *
+ * Deutsche Telekom AG and all other contributors /
+ * copyright owners license this file to you under the Apache
+ * License, Version 2.0 (the "License"); you may not use this
+ * file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package eu.tsystems.mms.tic.testframework.common;
+
+import eu.tsystems.mms.tic.testframework.events.ExecutionAbortEvent;
+import eu.tsystems.mms.tic.testframework.report.model.context.ExecutionContext;
+import eu.tsystems.mms.tic.testframework.report.utils.IExecutionContextController;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.eventbus.EventBus;
+
+/**
+ * triggers ExecutionAbortEvent in case of none existing report
+ * missing report is indicator of unexpected system.exit and reason to trigger report generation
+ */
+public class JVMExitHook extends Thread {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(JVMExitHook.class);
+
+    public void run(){
+
+        final IExecutionContextController contextController = Testerra.getInjector().getInstance(IExecutionContextController.class);
+        final ExecutionContext executionContext = contextController.getExecutionContext();
+
+        // trigger report generation only when no report exists, as shutdown hook is always executed no matter if normal finish or abortion
+        if (!executionContext.getReportModelGenerated()) {
+            LOGGER.info("Triggering report generation after unexpected abortion of test execution.");
+            final EventBus eventBus = Testerra.getEventBus();
+            eventBus.post(new ExecutionAbortEvent());
+        }
+    }
+}

core/src/main/java/eu/tsystems/mms/tic/testframework/common/Testerra.java

@@ -21,27 +21,12 @@
 
 package eu.tsystems.mms.tic.testframework.common;
 
-import com.google.common.eventbus.EventBus;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.google.inject.util.Modules;
 import eu.tsystems.mms.tic.testframework.events.ModulesInitializedEvent;
 import eu.tsystems.mms.tic.testframework.hooks.ModuleHook;
 import eu.tsystems.mms.tic.testframework.internal.BuildInformation;
 import eu.tsystems.mms.tic.testframework.logging.MethodContextLogAppender;
 import eu.tsystems.mms.tic.testframework.report.TesterraListener;
 import eu.tsystems.mms.tic.testframework.webdrivermanager.WebDriverRequest;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.core.LoggerContext;
-import org.apache.logging.log4j.core.config.Configurator;
-import org.apache.logging.log4j.core.config.DefaultConfiguration;
-import org.reflections.Reflections;
-import org.reflections.util.ConfigurationBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.InputStream;
@@ -55,6 +40,23 @@
 import java.util.TreeMap;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.logging.log4j.core.config.DefaultConfiguration;
+import org.reflections.Reflections;
+import org.reflections.util.ConfigurationBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.eventbus.EventBus;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.util.Modules;
+
 /**
  * This is the core main class where everything begins.
  * Using this method will initialize Testerra and all its modules.
@@ -140,6 +142,8 @@ public Object getDefault() {
         moduleHooks = new LinkedList<>();
         injector = initIoc();
         initHooks();
+
+        Runtime.getRuntime().addShutdownHook(new JVMExitHook());
         Runtime.getRuntime().addShutdownHook(new Thread(Testerra::shutdown));
     }
 

core/src/main/java/eu/tsystems/mms/tic/testframework/report/ExecutionEndListener.java

@@ -57,15 +57,20 @@ public final class ExecutionEndListener implements
     @Override
     @Subscribe
     public void onExecutionAbort(ExecutionAbortEvent event) {
-        contextController.getExecutionContext().setCrashed(true);
+        log().warn("Test execution was aborted. Test results may be incomplete.", Loggable.prompt);
+        contextController.getExecutionContext().setCrashed();
         finalizeExecutionContext();
+        log().info("Report generated after unexpected abortion of test execution.");
     }
 
     @Override
     @Subscribe
     public void onExecutionFinish(ExecutionFinishEvent event) {
         // set the testRunFinished flag
         finalizeExecutionContext();
+        log().info("Report generated after successful test execution.");
+        // set the reportGenerated flag
+        contextController.getExecutionContext().setReportModelGenerated();
     }
 
     private void finalizeExecutionContext() {

core/src/main/java/eu/tsystems/mms/tic/testframework/report/TesterraListener.java

@@ -41,6 +41,8 @@
 import eu.tsystems.mms.tic.testframework.report.model.context.MethodContext;
 import eu.tsystems.mms.tic.testframework.report.model.steps.TestStep;
 import eu.tsystems.mms.tic.testframework.report.utils.ExecutionContextController;
+import eu.tsystems.mms.tic.testframework.report.utils.IExecutionContextController;
+
 import org.apache.logging.log4j.core.LoggerContext;
 import org.testng.IConfigurable;
 import org.testng.IConfigurationListener;
@@ -332,8 +334,9 @@ private void pAfterInvocation(
 //        }
 
         final String methodName = getMethodName(testResult);
+        final IExecutionContextController executionContextController = Testerra.getInjector().getInstance(IExecutionContextController.class);
 
-        Optional<MethodContext> optionalMethodContext = ExecutionContextController.getMethodContextForThread();
+        Optional<MethodContext> optionalMethodContext = executionContextController.getCurrentMethodContext();
         MethodContext methodContext;
 
         if (!optionalMethodContext.isPresent()) {
@@ -371,6 +374,8 @@ public void generateReport(
         ExecutionFinishEvent event = new ExecutionFinishEvent()
                 .setSuites(suites)
                 .setXmlSuites(xmlSuites);
+
+        log().info("Triggering report generation after successful test execution.");
         Testerra.getEventBus().post(event);
     }
 

core/src/main/java/eu/tsystems/mms/tic/testframework/report/model/context/ExecutionContext.java

@@ -38,9 +38,10 @@ public class ExecutionContext extends AbstractContext {
      */
     public final RunConfig runConfig = new RunConfig();
     /**
-     * @deprecated Use {@link #isCrashed()} instead
-     */
-    public boolean crashed = false;
+     * flag to indicate whether report has been already generated, used by {@link eu.tsystems.mms.tic.testframework.common.JVMExitHook}
+     * */
+    private boolean reportModelGenerated;
+    private boolean crashed;
     private Queue<SessionContext> exclusiveSessionContexts;
     /**
      * @deprecated Use {@link #getEstimatedTestMethodCount()} instead
@@ -108,12 +109,23 @@ public RunConfig getRunConfig() {
         return this.runConfig;
     }
 
+    /**
+     * set reportModelGenerated flag, indicating already existing report
+     */
+    public synchronized void setReportModelGenerated() {
+        reportModelGenerated = true;
+    }
+
+    public boolean getReportModelGenerated() {
+        return reportModelGenerated;
+    }
+
     public boolean isCrashed() {
         return this.crashed;
     }
 
-    public void setCrashed(boolean crashed) {
-        this.crashed = crashed;
+    public void setCrashed() {
+        this.crashed = true;
     }
 
     public int getEstimatedTestMethodCount() {

docs/src/docs/execution.adoc

@@ -10,4 +10,3 @@ include::execution/execution-retry-analyzer.adoc[leveloffset=+1]
 include::execution/execution-watchdog.adoc[leveloffset=+1]
 include::execution/execution-annotation.adoc[leveloffset=+1]
 include::execution/execution-dry-run.adoc[leveloffset=+1]
-include::execution/execution-jvm-monitor.adoc[leveloffset=+1]

docs/src/docs/extending-testerra/events-and-listener.adoc

@@ -18,9 +18,9 @@ This event can be used to send test results to a test management system or issue
 
 | `ExecutionFinishEvent`
 |Called at the end of test run to trigger report generation and other output worker.
-
+[#ExecutionAbortEvent]
 |`ExecutionAbortEvent`
-|Called on test run abortion due to unclear circumstances like hanging sessions, JVM exit or other.
+|Called on test run abortion due to unclear circumstances like hanging sessions, JVM exit or other. Used to create report with existing execution information.
 
 |`InterceptMethodsEvent`
 |Called before suite execution. The events methods list provides a list of tests to execute. Read more about this in <<Intercept test method execution>>

docs/src/docs/index.adoc

@@ -32,6 +32,7 @@ include::reports.adoc[leveloffset=+1]
 include::modules.adoc[leveloffset=+1]
 include::utilities.adoc[leveloffset=+1]
 include::selenium4.adoc[leveloffset=+1]
+include::jvm.adoc[leveloffset=+1]
 
 '''
 = Extending Testerra

docs/src/docs/jvm.adoc

@@ -0,0 +1,4 @@
+= JVM
+
+include::jvm/jvm-monitor.adoc[leveloffset=+1]
+include::jvm/jvm-exit-hook.adoc[leveloffset=+1]

docs/src/docs/jvm/jvm-exit-hook.adoc

@@ -0,0 +1,7 @@
+[#JvmExitHook]
+= JVMExitHook
+
+The `JVMExitHook` was integrated to achieve the generation of a report with Report-NG in case an <<#ExecutionAbortion, aborted test run>>. With the start of a test execution the `JVMExitHook` is added as a shutdown hook to the JVM.
+
+When the JVM is about to stop, shutdown hooks are always called by default. This includes cases of abortion, all other system errors and normal finish. In the latter case a flag indicating an already created report is set true once in the process of the report generation.
+To avoid overriding the already existing report, the `JVMExitHook` only triggers when this flag is false, as this is the only indicator of an unexpected exit and a missing report. It then sends the <<#ExecutionAbortEvent, ExecutionAbortEvent>>, which is then caught by the corresponding listeners creating a report.

docs/src/docs/execution/execution-jvm-monitor.adoc → docs/src/docs/jvm/jvm-monitor.adoc

@@ -3,7 +3,7 @@ include::../properties/property-attributes.adoc[]
 
 The `JVMMonitor` is the Observer of the hardware utilization for memory and cpu.
 With the start of a test while using the `TesterraListener` the latter implicitly starts the `JVMMonitor`.
-Thus a concurrent thread for monitoring purposes only is initiated next to the actual test execution. Every ten seconds the following parameters are logged
+Thus a concurrent thread for monitoring purposes only is initiated next to the actual test execution. Every ten seconds the following parameters are logged at DEBUG Level.
 
 * JVM Memory usage      in MB
 * JVM Memory reserved   in MB

docs/src/docs/reports.adoc

@@ -12,3 +12,4 @@ include::reports/prioritymessages.adoc[leveloffset=+1]
 include::reports/layout-check.adoc[leveloffset=+1]
 include::reports/assertions.adoc[leveloffset=+1]
 include::reports/customizations.adoc[leveloffset=+1]
+include::reports/aborted-execution.adoc[leveloffset=+1]

docs/src/docs/reports/aborted-execution.adoc

@@ -0,0 +1,16 @@
+[#ExecutionAbortion]
+= Report of aborted test runs
+
+In case of unexpected abortion of a test execution the report is generated, too. . Due to abortion of still running test methods
+some information can be missing, like result states or execution time.
+
+Testerra additionally provides a mechanism to implicitly create a test report when an unexpected exit of the current execution occurred. All existing information created until the moment of the abortion are collected to generate a report.
+This might lead to missing as information like the execution time, result states and screenshots might be abandoned if running test methods have been interrupted.
+
+image::report-ng-24.png[align="center", alt="Report-NG states missing"]
+image::report-ng-25.png[align="center", alt="Report-NG execution time missing"]
+
+A hint indicating the possibly incomplete state of the report is shown as a warning in the section 'priority messages'.
+The report generation in case of abortion of the test execution is described in detail in the paragraph <<#JvmExitHook, JVMExitHook>>.
+
+image::report-ng-23.png[align="center", alt="Report-NG Aborted Exection"]

report-model/src/main/java/eu/tsystems/mms/tic/testframework/listeners/GenerateJUnitXML2ReportListener.java

@@ -22,23 +22,13 @@
 
 package eu.tsystems.mms.tic.testframework.listeners;
 
-import com.google.common.eventbus.Subscribe;
+import eu.tsystems.mms.tic.testframework.common.Testerra;
 import eu.tsystems.mms.tic.testframework.events.ExecutionFinishEvent;
 import eu.tsystems.mms.tic.testframework.events.MethodEndEvent;
 import eu.tsystems.mms.tic.testframework.events.TestStatusUpdateEvent;
 import eu.tsystems.mms.tic.testframework.logging.Loggable;
 import eu.tsystems.mms.tic.testframework.report.Report;
 import eu.tsystems.mms.tic.testframework.report.Status;
-import eu.tsystems.mms.tic.testframework.report.TesterraListener;
-import org.testng.ITestContext;
-import org.testng.ITestNGMethod;
-import org.testng.ITestResult;
-import org.testng.collections.Maps;
-import org.testng.collections.Sets;
-import org.testng.internal.Utils;
-import org.testng.reporters.XMLConstants;
-import org.testng.reporters.XMLStringBuffer;
-import org.testng.util.TimeUtils;
 
 import java.lang.reflect.Method;
 import java.net.InetAddress;
@@ -52,6 +42,18 @@
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.regex.Pattern;
 
+import org.testng.ITestContext;
+import org.testng.ITestNGMethod;
+import org.testng.ITestResult;
+import org.testng.collections.Maps;
+import org.testng.collections.Sets;
+import org.testng.internal.Utils;
+import org.testng.reporters.XMLConstants;
+import org.testng.reporters.XMLStringBuffer;
+import org.testng.util.TimeUtils;
+
+import com.google.common.eventbus.Subscribe;
+
 /**
  * Generates an JUnit XML report.
  * <p>
@@ -78,7 +80,7 @@ public class GenerateJUnitXML2ReportListener implements
 
     private ITestContext testngTestContext = null;
 
-    private Report report = TesterraListener.getReport();
+    private final Report report = Testerra.getInjector().getInstance(Report.class);
 
     static {
         ATTR_ESCAPES.put("&lt;", LESS);

report-model/src/main/java/eu/tsystems/mms/tic/testframework/listeners/GenerateTestNGXmlReportListener.java

@@ -22,11 +22,12 @@
 
 package eu.tsystems.mms.tic.testframework.listeners;
 
-import com.google.common.eventbus.Subscribe;
+import eu.tsystems.mms.tic.testframework.common.Testerra;
 import eu.tsystems.mms.tic.testframework.events.ExecutionFinishEvent;
 import eu.tsystems.mms.tic.testframework.logging.Loggable;
 import eu.tsystems.mms.tic.testframework.report.Report;
-import eu.tsystems.mms.tic.testframework.report.TesterraListener;
+
+import com.google.common.eventbus.Subscribe;
 
 /**
  * Generate TestNG result XML
@@ -35,7 +36,7 @@
  */
 public class GenerateTestNGXmlReportListener implements Loggable, ExecutionFinishEvent.Listener {
 
-    private Report report = TesterraListener.getReport();
+    private final Report report = Testerra.getInjector().getInstance(Report.class);
 
     @Override
     @Subscribe