Allow custom failure patterns in dmesg test check

docs/releases.rst

@@ -20,6 +20,10 @@ of the guest. See :ref:`command-variables` for details.
 New section :ref:`review` describing benefits and various forms of
 pull request reviews has been added to the :ref:`contribute` docs.
 
+The :ref:`dmesg test check<plugins/test-checks/dmesg>` can be
+configured to look for custom patterns in the output of ``dmesg``
+command, by setting its ``failure-pattern`` key.
+
 
 tmt-1.32.2
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/templates/plugins.rst.j2

@@ -46,16 +46,20 @@ Configuration
     {% if metadata.has_default %}
         {% set actual_default = metadata.materialized_default %}
 
-        {% if actual_default is sequence and not actual_default %}
-    Default: *not set*
-        {% elif actual_default is boolean %}
+        {% if actual_default is boolean %}
     Default: ``{{ actual_default | string | lower }}``
         {% elif actual_default is string %}
     Default: ``{{ actual_default }}``
         {% elif actual_default is integer %}
     Default: ``{{ actual_default }}``
         {% elif actual_default is none %}
     Default: *not set*
+        {% elif actual_default is sequence %}
+            {% if not actual_default %}
+    Default: *not set*
+            {% else %}
+    Default: {% for default_item in actual_default %}``{{ default_item.pattern | default(default_item) }}`` {% endfor %}
+            {% endif %}
         {% else %}
             {% set _ = LOGGER.warn("%s/%s.%s: could not render default value, '%s'" | format(STEP, PLUGIN_ID, field_name, actual_default)) %}
     Default: *could not render default value correctly*

tests/test/check/data/main.fmf

@@ -1,12 +1,18 @@
 /dmesg:
+    test: /bin/true
     check: dmesg
 
     /harmless:
-        test: /bin/true
 
     /segfault:
         test: echo Some segfault happened > /dev/kmsg
 
+    /custom-patterns:
+        check:
+          - how: dmesg
+            failure-pattern:
+              - 'Hypervisor detected'
+
 /avc:
   check:
     - how: avc

tests/test/check/test-dmesg.sh

@@ -18,24 +18,18 @@ rlJournalStart
 
         rlRun "results=$run/plan/execute/results.yaml"
         rlRun "harmless=$run/plan/execute/data/guest/default-0/dmesg/harmless-1"
-        rlRun "segfault=$run/plan/execute/data/guest/default-0/dmesg/segfault-2"
-        rlRun "dump_before=$harmless/checks/dmesg-before-test.txt"
-        rlRun "dump_after=$harmless/checks/dmesg-after-test.txt"
+        rlRun "segfault=$run/plan/execute/data/guest/default-0/dmesg/segfault-1"
+        rlRun "custom_patterns=$run/plan/execute/data/guest/default-0/dmesg/custom-patterns-1"
 
         rlRun "pushd data"
         rlRun "set -o pipefail"
     rlPhaseEnd
 
-    rlPhaseStartTest "Test dmesg check with $PROVISION_HOW"
-        # Segfault test only reproducible with virtual (needs root)
-        if [ "$PROVISION_HOW" = "virtual" ]; then
-            test_name=/dmesg
-        else
-            test_name=/dmesg/harmless
-        fi
-
-        rlRun "tmt run --id $run --scratch -a -vv provision -h $PROVISION_HOW test -n $test_name"
+    rlPhaseStartTest "Test dmesg check with $PROVISION_HOW in harmless run"
+        rlRun "dump_before=$harmless/checks/dmesg-before-test.txt"
+        rlRun "dump_after=$harmless/checks/dmesg-after-test.txt"
 
+        rlRun "tmt run --id $run --scratch -a -vv provision -h $PROVISION_HOW test -n /dmesg/harmless"
         rlRun "cat $results"
 
         if [ "$PROVISION_HOW" = "container" ]; then
@@ -48,28 +42,62 @@ rlJournalStart
 
             rlAssertExists "$dump_before"
             rlLogInfo "$(cat $dump_before)"
+
         fi
 
         if [ "$PROVISION_HOW" = "container" ]; then
             assert_check_result "dmesg as an after-test should skip with containers" "skip" "after-test" "harmless"
 
             rlAssertNotExists "$dump_after"
 
-        elif [ "$PROVISION_HOW" = "virtual" ]; then
+        else
+            assert_check_result "dmesg as an after-test should pass" "pass" "after-test" "harmless"
+
+            rlAssertExists "$dump_after"
+            rlLogInfo "$(cat $dump_after)"
+
+        fi
+    rlPhaseEnd
+
+    if [ "$PROVISION_HOW" = "virtual" ]; then
+        # Segfault test only reproducible with virtual (needs root)
+        rlPhaseStartTest "Test dmesg check with $PROVISION_HOW with a segfault"
+            rlRun "dump_before=$segfault/checks/dmesg-before-test.txt"
+            rlRun "dump_after=$segfault/checks/dmesg-after-test.txt"
+
+            rlRun "tmt run --id $run --scratch -a -vv provision -h $PROVISION_HOW test -n /dmesg/segfault"
+            rlRun "cat $results"
+
+            assert_check_result "dmesg as a before-test should pass" "pass" "before-test" "segfault"
+
+            rlAssertExists "$dump_before"
+            rlLogInfo "$(cat $dump_before)"
+
             assert_check_result "dmesg as an after-test should fail" "fail" "after-test" "segfault"
 
             rlAssertExists "$dump_after"
             rlLogInfo "$(cat $dump_after)"
+        rlPhaseEnd
 
-            rlAssertGrep "Some segfault happened" "$segfault/checks/dmesg-after-test.txt"
+        # Reproducible only with reliable dmesg content, e.g. after booting a fresh VM
+        rlPhaseStartTest "Test dmesg check with $PROVISION_HOW with custom patterns"
+            rlRun "dump_before=$custom_patterns/checks/dmesg-before-test.txt"
+            rlRun "dump_after=$custom_patterns/checks/dmesg-after-test.txt"
 
-        else
-            assert_check_result "dmesg as an after-test should pass" "pass" "after-test" "harmless"
+            rlRun "tmt run --id $run --scratch -a -vv provision -h $PROVISION_HOW test -n /dmesg/custom-patterns"
+            rlRun "cat $results"
+
+            assert_check_result "dmesg as a before-test should fail" "fail" "before-test" "custom-patterns"
+
+            rlAssertExists "$dump_before"
+            rlLogInfo "$(cat $dump_before)"
+
+            assert_check_result "dmesg as an after-test should pass" "pass" "after-test" "custom-patterns"
 
             rlAssertExists "$dump_after"
             rlLogInfo "$(cat $dump_after)"
-        fi
-    rlPhaseEnd
+        rlPhaseEnd
+    fi
 
     rlPhaseStartCleanup
         rlRun "popd"

tmt/checks/dmesg.py

@@ -1,23 +1,26 @@
+import dataclasses
 import datetime
 import re
+from re import Pattern
 from typing import TYPE_CHECKING, Optional
 
 import tmt.log
 import tmt.steps.execute
 import tmt.steps.provision
 import tmt.utils
-from tmt.checks import Check, CheckEvent, CheckPlugin, provides_check
+from tmt.checks import Check, CheckEvent, CheckPlugin, _RawCheck, provides_check
 from tmt.result import CheckResult, ResultOutcome
 from tmt.steps.provision import GuestCapability
-from tmt.utils import Path, render_run_exception_streams
+from tmt.utils import Path, field, render_run_exception_streams
 
 if TYPE_CHECKING:
     import tmt.base
     from tmt.steps.execute import TestInvocation
     from tmt.steps.provision import Guest
 
 TEST_POST_DMESG_FILENAME = 'dmesg-{event}.txt'
-FAILURE_PATTERNS = [
+
+DEFAULT_FAILURE_PATTERNS = [
     re.compile(pattern)
     for pattern in [
         r'Call Trace:',
@@ -26,37 +29,32 @@
     ]
 
 
-@provides_check('dmesg')
-class DmesgCheck(CheckPlugin[Check]):
-    """
-    Save the content of kernel ring buffer (aka "console") into a file.
-
-    The check saves one file before the test, and then again
-    when test finishes.
-
-    .. code-block:: yaml
-
-        check:
-          - name: dmesg
-
-    .. versionadded:: 1.28
-    """
+@dataclasses.dataclass
+class DmesgCheck(Check):
+    failure_pattern: list[Pattern[str]] = field(
+        default_factory=lambda: DEFAULT_FAILURE_PATTERNS[:],
+        help="""
+             List of regular expressions to look for in ``dmesg``
+             output. If any of patterns is found, ``dmesg`` check will
+             report ``fail`` result.
+             """,
+        normalize=tmt.utils.normalize_pattern_list,
+        exporter=lambda patterns: [pattern.pattern for pattern in patterns],
+        serialize=lambda patterns: [pattern.pattern for pattern in patterns],
+        unserialize=lambda serialized: [re.compile(pattern) for pattern in serialized]
+        )
 
-    _check_class = Check
+    # TODO: fix `to_spec` of `Check` to support nested serializables
+    def to_spec(self) -> _RawCheck:
+        spec = super().to_spec()
 
-    @classmethod
-    def essential_requires(
-            cls,
-            guest: 'Guest',
-            test: 'tmt.base.Test',
-            logger: tmt.log.Logger) -> list['tmt.base.DependencySimple']:
-        if not guest.facts.has_capability(GuestCapability.SYSLOG_ACTION_READ_ALL):
-            return []
+        spec['failure-pattern'] = [  # type: ignore[reportGeneralTypeIssues,typeddict-unknown-key,unused-ignore]
+            pattern.pattern for pattern in self.failure_pattern]
 
-        # Avoid circular imports
-        import tmt.base
+        return spec
 
-        return [tmt.base.DependencySimple('/usr/bin/dmesg')]
+    def to_minimal_spec(self) -> _RawCheck:
+        return self.to_spec()
 
     @classmethod
     def _fetch_dmesg(
@@ -90,9 +88,8 @@ def _test_output_logger(
 
         return guest.execute(script, log=_test_output_logger)
 
-    @classmethod
     def _save_dmesg(
-            cls,
+            self,
             invocation: 'TestInvocation',
             event: CheckEvent,
             logger: tmt.log.Logger) -> tuple[ResultOutcome, Path]:
@@ -106,7 +103,7 @@ def _save_dmesg(
         path = invocation.check_files_path / TEST_POST_DMESG_FILENAME.format(event=event.value)
 
         try:
-            dmesg_output = cls._fetch_dmesg(invocation.guest, logger)
+            dmesg_output = self._fetch_dmesg(invocation.guest, logger)
 
         except tmt.utils.RunError as exc:
             outcome = ResultOutcome.ERROR
@@ -115,7 +112,7 @@ def _save_dmesg(
         else:
             outcome = ResultOutcome.PASS
             output = dmesg_output.stdout or ''
-            if any(pattern.search(output) for pattern in FAILURE_PATTERNS):
+            if any(pattern.search(output) for pattern in self.failure_pattern):
                 outcome = ResultOutcome.FAIL
 
         invocation.phase.write(
@@ -124,26 +121,78 @@ def _save_dmesg(
 
         return outcome, path.relative_to(invocation.phase.step.workdir)
 
+
+@provides_check('dmesg')
+class Dmesg(CheckPlugin[DmesgCheck]):
+    """
+    Save the content of kernel ring buffer (aka "console") into a file.
+
+    The check saves one file before the test, and then again
+    when test finishes.
+
+    .. code-block:: yaml
+
+        check:
+          - how: dmesg
+
+    Check will identify patterns that signal kernel crashes and
+    core dumps, and when detected, it will report as failed result.
+    It is possible to define custom patterns:
+
+    .. code-block:: yaml
+
+        check:
+          - how: dmesg
+            failure-pattern:
+              # These are default patterns
+              - 'Call Trace:
+              - '\\ssegfault\\s'
+
+              # More patterns to look for
+              - '\\[Firmware Bug\\]'
+
+    .. versionadded:: 1.28
+
+    .. versionchanged:: 1.33
+       ``failure-pattern`` has been added.
+    """
+
+    _check_class = DmesgCheck
+
+    @classmethod
+    def essential_requires(
+            cls,
+            guest: 'Guest',
+            test: 'tmt.base.Test',
+            logger: tmt.log.Logger) -> list['tmt.base.DependencySimple']:
+        if not guest.facts.has_capability(GuestCapability.SYSLOG_ACTION_READ_ALL):
+            return []
+
+        # Avoid circular imports
+        import tmt.base
+
+        return [tmt.base.DependencySimple('/usr/bin/dmesg')]
+
     @classmethod
     def before_test(
             cls,
             *,
-            check: 'Check',
+            check: 'DmesgCheck',
             invocation: 'TestInvocation',
             environment: Optional[tmt.utils.Environment] = None,
             logger: tmt.log.Logger) -> list[CheckResult]:
         if not invocation.guest.facts.has_capability(GuestCapability.SYSLOG_ACTION_READ_ALL):
             return [CheckResult(name='dmesg', result=ResultOutcome.SKIP)]
 
-        outcome, path = cls._save_dmesg(invocation, CheckEvent.BEFORE_TEST, logger)
+        outcome, path = check._save_dmesg(invocation, CheckEvent.BEFORE_TEST, logger)
 
         return [CheckResult(name='dmesg', result=outcome, log=[path])]
 
     @classmethod
     def after_test(
             cls,
             *,
-            check: 'Check',
+            check: 'DmesgCheck',
             invocation: 'TestInvocation',
             environment: Optional[tmt.utils.Environment] = None,
             logger: tmt.log.Logger) -> list[CheckResult]:
@@ -153,6 +202,6 @@ def after_test(
         if invocation.hard_reboot_requested:
             return [CheckResult(name='dmesg', result=ResultOutcome.SKIP)]
 
-        outcome, path = cls._save_dmesg(invocation, CheckEvent.AFTER_TEST, logger)
+        outcome, path = check._save_dmesg(invocation, CheckEvent.AFTER_TEST, logger)
 
         return [CheckResult(name='dmesg', result=outcome, log=[path])]