test: Don't fail tests which call cloud-init as a command (#5209)

Implement verify_clean_boot() to ignore certain expected logs
in a platform-specific way.

tests/integration_tests/datasources/test_ec2_ipv6.py

@@ -2,13 +2,22 @@
 
 import pytest
 
+from cloudinit.util import should_log_deprecation
 from tests.integration_tests.instances import IntegrationInstance
 from tests.integration_tests.integration_settings import PLATFORM
-
-
-def _test_crawl(client, ip):
-    assert client.execute("cloud-init clean --logs").ok
-    assert client.execute("cloud-init init --local").ok
+from tests.integration_tests.util import get_feature_flag_value
+
+
+def _test_crawl(client: IntegrationInstance, ip: str):
+    return_code = (
+        2
+        if should_log_deprecation(
+            "24.3", get_feature_flag_value(client, "DEPRECATION_INFO_BOUNDARY")
+        )
+        else 0
+    )
+    assert client.execute("cloud-init clean --logs")
+    assert return_code == client.execute("cloud-init init --local").return_code
     log = client.read_from_file("/var/log/cloud-init.log")
     assert f"Using metadata source: '{ip}'" in log
     result = re.findall(r"Crawl of metadata service.* (\d+.\d+) seconds", log)

tests/integration_tests/test_upgrade.py

@@ -14,7 +14,7 @@
     IS_UBUNTU,
     MANTIC,
 )
-from tests.integration_tests.util import verify_clean_log
+from tests.integration_tests.util import verify_clean_boot, verify_clean_log
 
 LOG = logging.getLogger("integration_testing.test_upgrade")
 
@@ -81,11 +81,8 @@ def test_clean_boot_of_upgraded_package(session_cloud: IntegrationCloud):
         pre_cloud_blame = instance.execute("cloud-init analyze blame")
 
         # Ensure no issues pre-upgrade
-        log = instance.read_from_file("/var/log/cloud-init.log")
-        assert not json.loads(pre_result)["v1"]["errors"]
-
         try:
-            verify_clean_log(log)
+            verify_clean_boot(instance)
         except AssertionError:
             LOG.warning(
                 "There were errors/warnings/tracebacks pre-upgrade. "
@@ -122,10 +119,7 @@ def test_clean_boot_of_upgraded_package(session_cloud: IntegrationCloud):
         post_cloud_blame = instance.execute("cloud-init analyze blame")
 
         # Ensure no issues post-upgrade
-        assert not json.loads(pre_result)["v1"]["errors"]
-
-        log = instance.read_from_file("/var/log/cloud-init.log")
-        verify_clean_log(log)
+        verify_clean_boot(instance)
 
         # Ensure important things stayed the same
         assert pre_hostname == post_hostname

tests/integration_tests/util.py

@@ -14,6 +14,7 @@
 import pytest
 
 from cloudinit.subp import subp
+from tests.integration_tests.integration_settings import PLATFORM
 
 LOG = logging.getLogger("integration_testing.util")
 
@@ -70,7 +71,8 @@ def verify_clean_boot(
 ):
     """raise assertions if the client experienced unexpected warnings or errors
 
-    fail when an required error isn't found
+    Fail when a required error isn't found.
+    Expected warnings and errors are defined in this function.
 
     This function is similar to verify_clean_log, hence the similar name.
 
@@ -89,6 +91,77 @@ def verify_clean_boot(
     require_errors: Optional[list] = None,
     fail_when_expected_not_found: optional list of expected errors
     """
+
+    def append_or_create_list(
+        maybe_list: Optional[Union[List[str], bool]], value: str
+    ) -> List[str]:
+        """handle multiple types"""
+        if isinstance(maybe_list, list):
+            maybe_list.append(value)
+        elif maybe_list is None or isinstance(maybe_list, bool):
+            maybe_list = [value]
+        return maybe_list
+
+    # Define exceptions by matrix of platform and Ubuntu release
+    if "azure" == PLATFORM:
+        # Consistently on all Azure launches:
+        ignore_warnings = append_or_create_list(
+            ignore_warnings, "No lease found; using default endpoint"
+        )
+    elif "lxd_vm" == PLATFORM:
+        # Ubuntu lxd storage
+        ignore_warnings = append_or_create_list(
+            ignore_warnings, "thinpool by default on Ubuntu due to LP #1982780"
+        )
+        ignore_warnings = append_or_create_list(
+            ignore_warnings,
+            "Could not match supplied host pattern, ignoring:",
+        )
+    elif "oracle" == PLATFORM:
+        # LP: #1842752
+        ignore_errors = append_or_create_list(
+            ignore_warnings, "Stderr: RTNETLINK answers: File exists"
+        )
+        # LP: #1833446
+        ignore_warnings = append_or_create_list(
+            ignore_warnings,
+            "UrlError: 404 Client Error: Not Found for url: "
+            "http://169.254.169.254/latest/meta-data/",
+        )
+        # Oracle has a file in /etc/cloud/cloud.cfg.d that contains
+        # users:
+        # - default
+        # - name: opc
+        #   ssh_redirect_user: true
+        # This can trigger a warning about opc having no public key
+        ignore_warnings = append_or_create_list(
+            ignore_warnings,
+            "Unable to disable SSH logins for opc given ssh_redirect_user",
+        )
+
+    _verify_clean_boot(
+        instance,
+        ignore_warnings=ignore_warnings,
+        ignore_errors=ignore_errors,
+        require_warnings=require_warnings,
+        require_errors=require_errors,
+    )
+    # assert no Tracebacks
+    assert (
+        "0"
+        == instance.execute(
+            "grep --count Traceback /var/log/cloud-init.log"
+        ).stdout.strip()
+    ), "Unexpected traceback found in /var/log/cloud-init.log"
+
+
+def _verify_clean_boot(
+    instance: "IntegrationInstance",
+    ignore_warnings: Optional[Union[List[str], bool]] = None,
+    ignore_errors: Optional[Union[List[str], bool]] = None,
+    require_warnings: Optional[list] = None,
+    require_errors: Optional[list] = None,
+):
     ignore_errors = ignore_errors or []
     ignore_warnings = ignore_warnings or []
     require_errors = require_errors or []