tests: Add report, restructure

Add html report
Restructure tests, markers

Signed-off-by: Jorgen Kvalvaag <[email protected]>

Author: jorgenmk

.github/workflows/on_target.yml

@@ -3,44 +3,23 @@ name: Target tests
 on:
   workflow_call:
     inputs:
+      artifact_run_id:
+        description: The run ID of the workflow to fetch artifacts from
+        type: string
+        required: true
       artifact_fw_version:
+        description: The firmware version found under this run_id
         type: string
         required: true
-      artifact_run_id:
+      pytest_marker:
         type: string
         required: true
-      run_log_check_tests:
-        type: boolean
-        required: false
-        default: true
-      run_fota_tests:
-        type: boolean
-        required: false
-        default: true
-      run_fullmfwfota_test:
-        type: boolean
-        required: false
-        default: false
-      run_dfu_tests:
-        type: boolean
-        required: false
-        default: true
-      run_connectivity_bridge_tests:
-        type: boolean
-        required: false
-        default: true
-      run_wifi_location_tests:
-        type: boolean
-        required: false
-        default: true
-      run_modem_trace_tests:
-        type: boolean
-        required: false
-        default: true
-      run_ppk_tests:
-        type: boolean
+        default: "-m not slow"
+      pytest_path:
+        type: string
         required: false
-        default: false
+        default: tests
+
   workflow_dispatch:
     inputs:
       artifact_fw_version:
@@ -51,38 +30,16 @@ on:
         description: The run ID of the workflow to fetch artifacts from
         type: string
         required: true
-      run_log_check_tests:
-        type: boolean
-        required: true
-        default: true
-      run_fota_tests:
-        type: boolean
-        required: true
-        default: false
-      run_fullmfwfota_test:
-        type: boolean
-        required: true
-        default: false
-      run_dfu_tests:
-        type: boolean
-        required: true
-        default: false
-      run_connectivity_bridge_tests:
-        type: boolean
-        required: true
-        default: false
-      run_wifi_location_tests:
-        type: boolean
-        required: true
-        default: false
-      run_modem_trace_tests:
-        type: boolean
-        required: true
-        default: false
-      run_ppk_tests:
-        type: boolean
+      pytest_marker:
+        description: The pytest marker to run
+        type: string
         required: true
-        default: false
+        default: "-m not slow"
+      pytest_path:
+        description: Select test execution path
+        type: string
+        required: false
+        default: tests
 
 jobs:
   target_test:
@@ -135,107 +92,27 @@ jobs:
               --software-version ${{ inputs.artifact_fw_version }} \
               hello.nrfcloud.com-${{ inputs.artifact_fw_version }}-thingy91x-nrf91.elf
 
-      - name: Run UART tests
-        if: ${{ inputs.run_log_check_tests }}
+      - name: Tests
         working-directory: thingy91x-oob/tests/on_target
         run: |
           mkdir -p results
-          pytest -s -v -m "dut1 and uart" \
-            --junit-xml=results/test-results-uart.xml \
-            tests
+          pytest -s -v ${{ inputs.pytest_marker }} \
+            --junit-xml=results/test-results-dut1.xml \
+            --html=results/test-results-dut1.html --self-contained-html \
+            ${{ inputs.pytest_path }}
         env:
           SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-          LOG_FILENAME: oob_uart_test_log
-
-      - name: Run FOTA tests (standard)
-        if: ${{ inputs.run_fota_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut1 and fota" \
-            --junit-xml=results/test-results-fota.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-          IMEI: ${{ secrets.IMEI_DUT_1 }}
-          FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
-          LOG_FILENAME: oob_fota_test_log
-
-      - name: Run FOTA tests (FULLMFW)
-        if: ${{ inputs.run_fullmfwfota_test }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut1 and fullmfw_fota" \
-            --junit-xml=results/test-results-fullmfw-fota.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-          IMEI: ${{ secrets.IMEI_DUT_1 }}
-          FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
-          LOG_FILENAME: oob_fullmfw_fota_test_log
-
-      - name: Run DFU tests
-        if: ${{ inputs.run_dfu_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut2 and dfu" \
-            --junit-xml=results/test-results-dfu.xml \
-            tests
-        env:
           SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }}
           SEGGER_NRF91: ${{ secrets.SEGGER_DUT_2_NRF91 }}
+          IMEI: ${{ secrets.IMEI_DUT_1 }}
           UART_ID: ${{ secrets.UART_DUT_2 }}
-
-          LOG_FILENAME: oob_dfu_test_log
-
-      - name: Run Connectivity Bridge Test
-        if: ${{ inputs.run_connectivity_bridge_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut2 and conn_bridge" \
-            --junit-xml=results/test-results-connectivity-bridge.xml \
-            tests
-        env:
-          SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }}
-          UART_ID: ${{ secrets.UART_DUT_2 }}
-          LOG_FILENAME: oob_conn_bridge_test_log
-
-      - name: Run Wi-Fi location tests
-        if: ${{ inputs.run_wifi_location_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          mkdir -p results
-          pytest -s -v -m "dut1 and wifi" \
-            --junit-xml=results/test-results-wifi-location.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-
-      - name: Run modem trace test
-        if: ${{ inputs.run_modem_trace_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          mkdir -p results
-          pytest -s -v -m "dut1 and traces" \
-            --junit-xml=results/test-results-traces-location.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-
-      - name: Run PPK tests
-        if: ${{ inputs.run_ppk_tests }}
-        continue-on-error: true
-        id: ppk_test
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          mkdir -p results
-          pytest -s -v -m dut_ppk \
-            --junit-xml=results/test-results-ppk.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_PPK }}
+          FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
+          LOG_FILENAME: oob_uart_test_log
+          TEST_REPORT_NAME: OOB Firwmare Functional Test Report
+          DUT1_HW_REVISION: ${{ vars.DUT1_HW_REVISION }}
+          SEGGER_PPK: ${{ secrets.SEGGER_DUT_PPK }}
 
       - name: Commit and Push Badge File to gh-pages Branch
-        if: ${{ inputs.run_ppk_tests }}
         continue-on-error: true
         working-directory: thingy91x-oob
         env:
@@ -251,6 +128,15 @@ jobs:
           fail-on-empty: true
           title: OOB FW Test Results
 
+      - name: Create Report Artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        id: artifact-report
+        with:
+          name: test-report
+          path: |
+            thingy91x-oob/tests/on_target/results/*.html
+
       - name: Push log files to artifacts
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/test.yml

@@ -3,34 +3,11 @@ name: Build and Test
 on:
   workflow_dispatch:
     inputs:
-      run_fota_tests:
-        type: boolean
+      pytest_marker:
+        type: string
         required: true
-        default: true
-      run_fullmfwfota_test:
-        type: boolean
-        required: true
-        default: false
-      run_dfu_tests:
-        type: boolean
-        required: true
-        default: true
-      run_connectivity_bridge_tests:
-        type: boolean
-        required: true
-        default: true
-      run_wifi_location_tests:
-        type: boolean
-        required: true
-        default: true
-      run_modem_trace_tests:
-        type: boolean
-        required: true
-        default: true
-      run_ppk_tests:
-        type: boolean
-        required: true
-        default: false
+        default: "-m not slow"
+
   schedule:
     - cron: "0 0 * * *"
   push:
@@ -58,43 +35,4 @@ jobs:
     with:
       artifact_fw_version: ${{ needs.build.outputs.version }}
       artifact_run_id: ${{ needs.build.outputs.run_id }}
-      run_fota_tests: >-
-        ${{
-          github.event_name == 'schedule' ||
-          github.event_name == 'push' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_fota_tests == 'true')
-        }}
-      run_fullmfwfota_test: >-
-        ${{
-          github.event_name == 'schedule' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_fullmfwfota_test == 'true')
-        }}
-      run_dfu_tests: >-
-        ${{
-          github.event_name == 'schedule' ||
-          github.event_name == 'push' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_dfu_tests == 'true')
-        }}
-      run_connectivity_bridge_tests: >-
-        ${{
-          github.event_name == 'schedule' ||
-          github.event_name == 'push' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_connectivity_bridge_tests == 'true')
-        }}
-      run_wifi_location_tests: >-
-        ${{
-            github.event_name == 'schedule' ||
-            github.event_name == 'push' ||
-            (github.event_name == 'workflow_dispatch' && github.event.inputs.run_wifi_location_tests == 'true')
-        }}
-      run_modem_trace_tests: >-
-        ${{
-          github.event_name == 'schedule' ||
-          github.event_name == 'push' ||
-          (github.event_name == 'workflow_dispatch' && github.event.inputs.run_modem_trace_tests == 'true')
-        }}
-      run_ppk_tests: >-
-        ${{
-            github.event_name == 'schedule' ||
-            (github.event_name == 'workflow_dispatch' && github.event.inputs.run_ppk_tests == 'true')
-        }}
+      pytest_marker: ${{ github.event.inputs.pytest_marker || '' }}

tests/on_target/requirements.txt

@@ -1,4 +1,5 @@
 pytest
+pytest-html
 pyserial
 termcolor
 pyusb

tests/on_target/tests/conftest.py

@@ -5,13 +5,13 @@
 
 import os
 import re
-import pytest
 import types
+import pytest
+import subprocess
+from pytest_metadata.plugin import metadata_key
 from utils.flash_tools import recover_device
 from utils.uart import Uart, UartBinary
 from utils.hellonrfcloud_fota import HelloNrfCloudFOTA
-import sys
-sys.path.append(os.getcwd())
 from utils.logger import get_logger
 
 logger = get_logger()
@@ -23,6 +23,41 @@
 FOTADEVICE_IMEI = os.getenv('IMEI')
 FOTADEVICE_FINGERPRINT = os.getenv('FINGERPRINT')
 
+def pytest_html_report_title(report):
+    report.title = os.getenv("TEST_REPORT_NAME", "OOB Test Report")
+
+
+def check_output(cmd, regexp):
+    p = subprocess.check_output(cmd.split()).decode().strip()
+    match = re.search(regexp, p)
+    return match.group(1)
+
+def pytest_configure(config):
+    config.stash[metadata_key]["Board"] = "thingy91x"
+    config.stash[metadata_key]["Board revision"] = os.getenv("DUT1_HW_REVISION", "Undefined")
+    config.stash[metadata_key]["nrfutil version"] = check_output("nrfutil --version", r"nrfutil ([0-9\.]+)")
+    config.stash[metadata_key]["nrfutil-device version"] = check_output(
+        "nrfutil device --version",
+        r"nrfutil-device ([0-9\.]+)"
+    )
+    config.stash[metadata_key]["SEGGER JLink version"] = check_output(
+        "nrfutil device --version",
+        r"SEGGER J-Link version: JLink_V([0-9a-z\.]+)"
+    )
+
+# Add column to table for test specification
+def pytest_html_results_table_header(cells):
+    cells.insert(2, "<th>Specification</th>")
+
+def pytest_html_results_table_row(report, cells):
+    cells.insert(2, f"<td>{report.description}</td>")
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    outcome = yield
+    report = outcome.get_result()
+    report.description = str(item.function.__doc__)
+
 def get_uarts():
     base_path = "/dev/serial/by-id"
     try:
@@ -61,7 +96,7 @@ def t91x_board():
     log_uart_string = all_uarts[0]
     uart = Uart(log_uart_string, timeout=UART_TIMEOUT)
     fota = HelloNrfCloudFOTA(device_id=f"oob-{FOTADEVICE_IMEI}", \
-                                    fingerprint=FOTADEVICE_FINGERPRINT)
+                             fingerprint=FOTADEVICE_FINGERPRINT)
 
     yield types.SimpleNamespace(
 		uart=uart,