Skip to content

Commit 89aa9f9

Browse files
jorgenmkjorgenmk
committed
tests: Add report, restructure
Add html report Restructure tests, markers Signed-off-by: Jorgen Kvalvaag <[email protected]>
1 parent 7c9ef06 commit 89aa9f9

File tree

13 files changed

+163
-266
lines changed

13 files changed

+163
-266
lines changed

‎.github/workflows/on_target.yml

+41-155
Original file line numberDiff line numberDiff line change
@@ -3,44 +3,23 @@ name: Target tests
33
on:
44
workflow_call:
55
inputs:
6+
artifact_run_id:
7+
description: The run ID of the workflow to fetch artifacts from
8+
type: string
9+
required: true
610
artifact_fw_version:
11+
description: The firmware version found under this run_id
712
type: string
813
required: true
9-
artifact_run_id:
14+
pytest_marker:
1015
type: string
1116
required: true
12-
run_log_check_tests:
13-
type: boolean
14-
required: false
15-
default: true
16-
run_fota_tests:
17-
type: boolean
18-
required: false
19-
default: true
20-
run_fullmfwfota_test:
21-
type: boolean
22-
required: false
23-
default: false
24-
run_dfu_tests:
25-
type: boolean
26-
required: false
27-
default: true
28-
run_connectivity_bridge_tests:
29-
type: boolean
30-
required: false
31-
default: true
32-
run_wifi_location_tests:
33-
type: boolean
34-
required: false
35-
default: true
36-
run_modem_trace_tests:
37-
type: boolean
38-
required: false
39-
default: true
40-
run_ppk_tests:
41-
type: boolean
17+
default: -m "not slow"
18+
pytest_path:
19+
type: string
4220
required: false
43-
default: false
21+
default: tests
22+
4423
workflow_dispatch:
4524
inputs:
4625
artifact_fw_version:
@@ -51,38 +30,16 @@ on:
5130
description: The run ID of the workflow to fetch artifacts from
5231
type: string
5332
required: true
54-
run_log_check_tests:
55-
type: boolean
56-
required: true
57-
default: true
58-
run_fota_tests:
59-
type: boolean
60-
required: true
61-
default: false
62-
run_fullmfwfota_test:
63-
type: boolean
64-
required: true
65-
default: false
66-
run_dfu_tests:
67-
type: boolean
68-
required: true
69-
default: false
70-
run_connectivity_bridge_tests:
71-
type: boolean
72-
required: true
73-
default: false
74-
run_wifi_location_tests:
75-
type: boolean
76-
required: true
77-
default: false
78-
run_modem_trace_tests:
79-
type: boolean
80-
required: true
81-
default: false
82-
run_ppk_tests:
83-
type: boolean
33+
pytest_marker:
34+
description: The pytest marker to run
35+
type: string
8436
required: true
85-
default: false
37+
default: -m "not slow"
38+
pytest_path:
39+
description: Select test execution path
40+
type: string
41+
required: false
42+
default: tests
8643

8744
jobs:
8845
target_test:
@@ -135,107 +92,27 @@ jobs:
13592
--software-version ${{ inputs.artifact_fw_version }} \
13693
hello.nrfcloud.com-${{ inputs.artifact_fw_version }}-thingy91x-nrf91.elf
13794
138-
- name: Run UART tests
139-
if: ${{ inputs.run_log_check_tests }}
95+
- name: Tests
14096
working-directory: thingy91x-oob/tests/on_target
14197
run: |
14298
mkdir -p results
143-
pytest -s -v -m "dut1 and uart" \
144-
--junit-xml=results/test-results-uart.xml \
145-
tests
146-
env:
147-
SEGGER: ${{ secrets.SEGGER_DUT_1 }}
148-
LOG_FILENAME: oob_uart_test_log
149-
150-
- name: Run FOTA tests (standard)
151-
if: ${{ inputs.run_fota_tests }}
152-
working-directory: thingy91x-oob/tests/on_target
153-
run: |
154-
pytest -s -v -m "dut1 and fota" \
155-
--junit-xml=results/test-results-fota.xml \
156-
tests
157-
env:
158-
SEGGER: ${{ secrets.SEGGER_DUT_1 }}
159-
IMEI: ${{ secrets.IMEI_DUT_1 }}
160-
FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
161-
LOG_FILENAME: oob_fota_test_log
162-
163-
- name: Run FOTA tests (FULLMFW)
164-
if: ${{ inputs.run_fullmfwfota_test }}
165-
working-directory: thingy91x-oob/tests/on_target
166-
run: |
167-
pytest -s -v -m "dut1 and fullmfw_fota" \
168-
--junit-xml=results/test-results-fullmfw-fota.xml \
169-
tests
99+
pytest -s -v ${{ inputs.pytest_marker }} \
100+
--junit-xml=results/test-results-dut1.xml \
101+
--html=results/test-results-dut1.html --self-contained-html \
102+
${{ inputs.pytest_path }}
170103
env:
171104
SEGGER: ${{ secrets.SEGGER_DUT_1 }}
172-
IMEI: ${{ secrets.IMEI_DUT_1 }}
173-
FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
174-
LOG_FILENAME: oob_fullmfw_fota_test_log
175-
176-
- name: Run DFU tests
177-
if: ${{ inputs.run_dfu_tests }}
178-
working-directory: thingy91x-oob/tests/on_target
179-
run: |
180-
pytest -s -v -m "dut2 and dfu" \
181-
--junit-xml=results/test-results-dfu.xml \
182-
tests
183-
env:
184105
SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }}
185106
SEGGER_NRF91: ${{ secrets.SEGGER_DUT_2_NRF91 }}
186-
UART_ID: ${{ secrets.UART_DUT_2 }}
187-
188-
LOG_FILENAME: oob_dfu_test_log
189-
190-
- name: Run Connectivity Bridge Test
191-
if: ${{ inputs.run_connectivity_bridge_tests }}
192-
working-directory: thingy91x-oob/tests/on_target
193-
run: |
194-
pytest -s -v -m "dut2 and conn_bridge" \
195-
--junit-xml=results/test-results-connectivity-bridge.xml \
196-
tests
197-
env:
198-
SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }}
199-
UART_ID: ${{ secrets.UART_DUT_2 }}
200-
LOG_FILENAME: oob_conn_bridge_test_log
201-
202-
- name: Run Wi-Fi location tests
203-
if: ${{ inputs.run_wifi_location_tests }}
204-
working-directory: thingy91x-oob/tests/on_target
205-
run: |
206-
mkdir -p results
207-
pytest -s -v -m "dut1 and wifi" \
208-
--junit-xml=results/test-results-wifi-location.xml \
209-
tests
210-
env:
211-
SEGGER: ${{ secrets.SEGGER_DUT_1 }}
212-
213-
- name: Run modem trace test
214-
if: ${{ inputs.run_modem_trace_tests }}
215-
working-directory: thingy91x-oob/tests/on_target
216-
run: |
217-
mkdir -p results
218-
pytest -s -v -m "dut1 and traces" \
219-
--junit-xml=results/test-results-traces-location.xml \
220-
tests
221-
env:
222-
SEGGER: ${{ secrets.SEGGER_DUT_1 }}
223-
224-
- name: Run PPK tests
225-
if: ${{ inputs.run_ppk_tests }}
226-
continue-on-error: true
227-
id: ppk_test
228-
working-directory: thingy91x-oob/tests/on_target
229-
run: |
230-
mkdir -p results
231-
pytest -s -v -m dut_ppk \
232-
--junit-xml=results/test-results-ppk.xml \
233-
tests
234-
env:
235-
SEGGER: ${{ secrets.SEGGER_DUT_PPK }}
107+
IMEI: ${{ secrets.IMEI_DUT_1 }}
108+
UART_ID_DUT_2: ${{ secrets.UART_DUT_2 }}
109+
FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
110+
LOG_FILENAME: oob_uart_test_log
111+
TEST_REPORT_NAME: OOB Firwmare Test Report
112+
DUT1_HW_REVISION: ${{ vars.DUT1_HW_REVISION }}
113+
SEGGER_PPK: ${{ secrets.SEGGER_DUT_PPK }}
236114

237115
- name: Commit and Push Badge File to gh-pages Branch
238-
if: ${{ inputs.run_ppk_tests }}
239116
continue-on-error: true
240117
working-directory: thingy91x-oob
241118
env:
@@ -251,6 +128,15 @@ jobs:
251128
fail-on-empty: true
252129
title: OOB FW Test Results
253130

131+
- name: Create Report Artifact
132+
if: always()
133+
uses: actions/upload-artifact@v4
134+
id: artifact-report
135+
with:
136+
name: test-report
137+
path: |
138+
thingy91x-oob/tests/on_target/results/*.html
139+
254140
- name: Push log files to artifacts
255141
if: always()
256142
uses: actions/upload-artifact@v4

‎.github/workflows/test.yml

+5-67
Original file line numberDiff line numberDiff line change
@@ -3,34 +3,11 @@ name: Build and Test
33
on:
44
workflow_dispatch:
55
inputs:
6-
run_fota_tests:
7-
type: boolean
6+
pytest_marker:
7+
type: string
88
required: true
9-
default: true
10-
run_fullmfwfota_test:
11-
type: boolean
12-
required: true
13-
default: false
14-
run_dfu_tests:
15-
type: boolean
16-
required: true
17-
default: true
18-
run_connectivity_bridge_tests:
19-
type: boolean
20-
required: true
21-
default: true
22-
run_wifi_location_tests:
23-
type: boolean
24-
required: true
25-
default: true
26-
run_modem_trace_tests:
27-
type: boolean
28-
required: true
29-
default: true
30-
run_ppk_tests:
31-
type: boolean
32-
required: true
33-
default: false
9+
default: -m "not slow"
10+
3411
schedule:
3512
- cron: "0 0 * * *"
3613
push:
@@ -58,43 +35,4 @@ jobs:
5835
with:
5936
artifact_fw_version: ${{ needs.build.outputs.version }}
6037
artifact_run_id: ${{ needs.build.outputs.run_id }}
61-
run_fota_tests: >-
62-
${{
63-
github.event_name == 'schedule' ||
64-
github.event_name == 'push' ||
65-
(github.event_name == 'workflow_dispatch' && github.event.inputs.run_fota_tests == 'true')
66-
}}
67-
run_fullmfwfota_test: >-
68-
${{
69-
github.event_name == 'schedule' ||
70-
(github.event_name == 'workflow_dispatch' && github.event.inputs.run_fullmfwfota_test == 'true')
71-
}}
72-
run_dfu_tests: >-
73-
${{
74-
github.event_name == 'schedule' ||
75-
github.event_name == 'push' ||
76-
(github.event_name == 'workflow_dispatch' && github.event.inputs.run_dfu_tests == 'true')
77-
}}
78-
run_connectivity_bridge_tests: >-
79-
${{
80-
github.event_name == 'schedule' ||
81-
github.event_name == 'push' ||
82-
(github.event_name == 'workflow_dispatch' && github.event.inputs.run_connectivity_bridge_tests == 'true')
83-
}}
84-
run_wifi_location_tests: >-
85-
${{
86-
github.event_name == 'schedule' ||
87-
github.event_name == 'push' ||
88-
(github.event_name == 'workflow_dispatch' && github.event.inputs.run_wifi_location_tests == 'true')
89-
}}
90-
run_modem_trace_tests: >-
91-
${{
92-
github.event_name == 'schedule' ||
93-
github.event_name == 'push' ||
94-
(github.event_name == 'workflow_dispatch' && github.event.inputs.run_modem_trace_tests == 'true')
95-
}}
96-
run_ppk_tests: >-
97-
${{
98-
github.event_name == 'schedule' ||
99-
(github.event_name == 'workflow_dispatch' && github.event.inputs.run_ppk_tests == 'true')
100-
}}
38+
pytest_marker: ${{ github.event.inputs.pytest_marker || '' }}

‎tests/on_target/requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
pytest
2+
pytest-html
23
pyserial
34
termcolor
45
pyusb

0 commit comments

Comments
 (0)