feat: update support bundle to adapt change for mqttbroker (Azure#256)

azext_edge/edge/_help.py

@@ -15,7 +15,7 @@
     COMPAT_CLUSTER_CONFIG_APIS,
     COMPAT_DATA_PROCESSOR_APIS,
     COMPAT_DEVICEREGISTRY_APIS,
-    COMPAT_MQ_APIS,
+    COMPAT_MQTT_BROKER_APIS,
     COMPAT_OPCUA_APIS,
     COMPAT_ORC_APIS,
 )
@@ -51,7 +51,7 @@ def load_iotops_help():
         short-summary: Creates a standard support bundle zip archive for use in troubleshooting and diagnostics.
         long-summary: |
             {{Supported service APIs}}
-            - {COMPAT_MQ_APIS.as_str()}
+            - {COMPAT_MQTT_BROKER_APIS.as_str()}
             - {COMPAT_OPCUA_APIS.as_str()}
             - {COMPAT_DATA_PROCESSOR_APIS.as_str()}
             - {COMPAT_ORC_APIS.as_str()}
@@ -91,7 +91,7 @@ def load_iotops_help():
             - {COMPAT_AKRI_APIS.as_str()}
             - {COMPAT_DATA_PROCESSOR_APIS.as_str()}
             - {COMPAT_DEVICEREGISTRY_APIS.as_str()}
-            - {COMPAT_MQ_APIS.as_str()}
+            - {COMPAT_MQTT_BROKER_APIS.as_str()}
             - {COMPAT_OPCUA_APIS.as_str()}
 
         examples:

azext_edge/edge/commands_edge.py

@@ -57,7 +57,7 @@ def check(
     post_deployment_checks: Optional[bool] = None,
     as_object=None,
     context_name=None,
-    ops_service: str = "mq",
+    ops_service: str = OpsServiceType.mq.value,
     resource_kinds: List[str] = None,
     resource_name: str = None,
 ) -> Union[Dict[str, Any], None]:

azext_edge/edge/common.py

@@ -140,7 +140,7 @@ class OpsServiceType(ListableEnum):
     """
 
     auto = "auto"
-    mq = "mq"
+    mq = "broker"
     opcua = "opcua"
     dataprocessor = "dataprocessor"
     orc = "orc"
@@ -149,6 +149,16 @@ class OpsServiceType(ListableEnum):
     # TODO: re-enable billing once service is available post 0.6.0 release
     # billing = "billing"
 
+    @classmethod
+    def list_check_services(cls):
+        return [
+            cls.mq.value,
+            cls.opcua.value,
+            cls.dataprocessor.value,
+            cls.akri.value,
+            cls.deviceregistry.value,
+        ]
+
 
 class ResourceProviderMapping(ListableEnum):
     """

azext_edge/edge/params.py

@@ -148,7 +148,7 @@ def load_iotops_arguments(self, _):
         context.argument(
             "ops_service",
             options_list=["--ops-service", "--svc"],
-            choices=CaseInsensitiveList(["akri", "dataprocessor", "deviceregistry", "mq", "opcua"]),
+            choices=CaseInsensitiveList(OpsServiceType.list_check_services()),
             help="The IoT Operations service deployment that will be evaluated.",
         )
         context.argument(

azext_edge/edge/providers/edge_api/__init__.py

@@ -6,7 +6,7 @@
 
 from .base import EdgeResourceApi, EdgeApiManager
 from .clusterconfig import CLUSTER_CONFIG_API_V1
-from .mq import MQ_ACTIVE_API, MQ_API_V1B1, MqResourceKinds
+from .mq import MQ_ACTIVE_API, MQTT_BROKER_API_V1B1, MqResourceKinds
 from .dataprocessor import DATA_PROCESSOR_API_V1, DataProcessorResourceKinds
 from .opcua import OPCUA_API_V1, OpcuaResourceKinds
 from .orc import ORC_API_V1, OrcResourceKinds
@@ -22,7 +22,7 @@
     "EdgeApiManager",
     "MqResourceKinds",
     "MQ_ACTIVE_API",
-    "MQ_API_V1B1",
+    "MQTT_BROKER_API_V1B1",
     "OpcuaResourceKinds",
     "OPCUA_API_V1",
     "OrcResourceKinds",

azext_edge/edge/providers/edge_api/mq.py

@@ -21,12 +21,13 @@ class MqResourceKinds(ListableEnum):
     DATALAKE_CONNECTOR_TOPIC_MAP = "datalakeconnectortopicmap"
     KAFKA_CONNECTOR = "kafkaconnector"
     KAFKA_CONNECTOR_TOPIC_MAP = "kafkaconnectortopicmap"
-    IOT_HUB_CONNECTOR = "iothubconnector"
-    IOT_HUB_CONNECTOR_ROUTE_MAP = "iothubconnectorroutesmap"
 
 
-MQ_API_V1B1 = EdgeResourceApi(
-    group="mq.iotoperations.azure.com", version="v1beta1", moniker="mq", label="microsoft-iotoperations-mq"
+MQTT_BROKER_API_V1B1 = EdgeResourceApi(
+    group="mqttbroker.iotoperations.azure.com",
+    version="v1beta1",
+    moniker="broker",
+    label="microsoft-iotoperations-mq",
 )
 
-MQ_ACTIVE_API = MQ_API_V1B1
+MQ_ACTIVE_API = MQTT_BROKER_API_V1B1

azext_edge/edge/providers/support/base.py

@@ -140,7 +140,6 @@ def process_v1_pods(
 
 def process_deployments(
     directory_path: str,
-    return_namespaces: bool = False,
     field_selector: Optional[str] = None,
     label_selector: Optional[str] = None,
     prefix_names: Optional[List[str]] = None,
@@ -151,29 +150,18 @@ def process_deployments(
     deployments: V1DeploymentList = v1_apps.list_deployment_for_all_namespaces(
         label_selector=label_selector, field_selector=field_selector
     )
-    namespace_pods_work = {}
 
-    processed = _process_kubernetes_resources(
+    return _process_kubernetes_resources(
         directory_path=directory_path,
         resources=deployments,
         prefix_names=prefix_names,
         kind=BundleResourceKind.deployment.value,
     )
 
-    for deployment in deployments.items:
-        deployment_namespace: str = deployment.metadata.namespace
-
-        if deployment_namespace not in namespace_pods_work:
-            namespace_pods_work[deployment_namespace] = True
-
-    if return_namespaces:
-        return processed, namespace_pods_work
-
-    return processed
-
 
 def process_statefulset(
     directory_path: str,
+    return_namespaces: bool = False,
     field_selector: Optional[str] = None,
     label_selector: Optional[str] = None,
 ) -> List[dict]:
@@ -183,13 +171,25 @@ def process_statefulset(
     statefulsets: V1StatefulSetList = v1_apps.list_stateful_set_for_all_namespaces(
         label_selector=label_selector, field_selector=field_selector
     )
+    namespace_pods_work = {}
 
-    return _process_kubernetes_resources(
+    processed = _process_kubernetes_resources(
         directory_path=directory_path,
         resources=statefulsets,
         kind=BundleResourceKind.statefulset.value,
     )
 
+    for statefulset in statefulsets.items:
+        statefulset_namespace: str = statefulset.metadata.namespace
+
+        if statefulset_namespace not in namespace_pods_work:
+            namespace_pods_work[statefulset_namespace] = True
+
+    if return_namespaces:
+        return processed, namespace_pods_work
+
+    return processed
+
 
 def process_services(
     directory_path: str,

azext_edge/edge/providers/support/mq.py

@@ -10,7 +10,7 @@
 
 from knack.log import get_logger
 
-from azext_edge.edge.common import AIO_MQ_OPERATOR, AIO_MQ_RESOURCE_PREFIX
+from azext_edge.edge.common import AIO_MQ_RESOURCE_PREFIX
 from azext_edge.edge.providers.edge_api.mq import MqResourceKinds
 
 from ..edge_api import MQ_ACTIVE_API, EdgeResourceApi
@@ -19,7 +19,6 @@
     DAY_IN_SECONDS,
     assemble_crd_work,
     get_mq_namespaces,
-    process_deployments,
     process_replicasets,
     process_services,
     process_statefulset,
@@ -31,14 +30,9 @@
 
 # TODO: @jiacju - will remove old labels once new labels are stabled
 MQ_APP_LABELS = [
-    "broker",  # aio-mq-dmqtt-frontend, aio-mq-dmqtt-backend, aio-mq-dmqtt-authentication
-    "diagnostics",  # aio-mq-diagnostics-service
-    "health-manager",  # aio-mq-dmqtt-health-manager
-    "aio-mq-operator",
     "aio-mq-mqttbridge",
     "aio-mq-datalake",
     "aio-mq-kafka-connector",
-    "aio-mq-iothub-connector",
 ]
 
 MQ_LABEL = f"app in ({','.join(MQ_APP_LABELS)})"
@@ -87,50 +81,11 @@ def fetch_diagnostic_traces():
     return result
 
 
-def fetch_deployments():
-    processed, namespaces = process_deployments(
-        directory_path=MQ_DIRECTORY_PATH, label_selector=MQ_LABEL, return_namespaces=True
-    )
-    # aio-mq-operator deployment has no app label
-    operators, operator_namespaces = process_deployments(
-        directory_path=MQ_DIRECTORY_PATH, field_selector=f"metadata.name={AIO_MQ_OPERATOR}", return_namespaces=True
-    )
-    processed.extend(operators)
-
-    operators_v2, operator_namespaces_v2 = process_deployments(
-        directory_path=MQ_DIRECTORY_PATH, label_selector=MQ_NAME_LABEL, return_namespaces=True
-    )
-    processed.extend(operators_v2)
-
-    for namespace in {**namespaces, **operator_namespaces, **operator_namespaces_v2}:
-        metrics: dict = fetch_diagnostic_metrics(namespace)
-        if metrics:
-            processed.append(metrics)
-
-        # TODO: @digimaun - enable after support for disabling check polling UX.
-        # try:
-        #     checks = run_checks(namespace=namespace)
-        #     checks_data = {
-        #         "data": checks,
-        #         "zinfo": f"{MQ_ACTIVE_API.moniker}/{namespace}/checks.yaml",
-        #     }
-        #     processed.append(checks_data)
-        # except Exception:
-        #     logger.debug(f"Unable to run checks against namespace {namespace}.")
-
-    return processed
-
-
 def fetch_statefulsets():
-    processed = process_statefulset(
+    processed, namespaces = process_statefulset(
         directory_path=MQ_DIRECTORY_PATH,
-        label_selector=MQ_LABEL,
-    )
-    processed.extend(
-        process_statefulset(
-            directory_path=MQ_DIRECTORY_PATH,
-            label_selector=MQ_NAME_LABEL,
-        )
+        label_selector=MQ_NAME_LABEL,
+        return_namespaces=True,
     )
 
     # bridge connector stateful sets have no labels
@@ -150,6 +105,11 @@ def fetch_statefulsets():
         )
         processed.extend(stateful_set)
 
+    for namespace in namespaces:
+        metrics = fetch_diagnostic_metrics(namespace)
+        if metrics:
+            processed.append(metrics)
+
     return processed
 
 
@@ -204,7 +164,6 @@ def fetch_pods(since_seconds: int = DAY_IN_SECONDS):
     "statefulsets": fetch_statefulsets,
     "replicasets": fetch_replicasets,
     "services": fetch_services,
-    "deployments": fetch_deployments,
 }
 
 

azext_edge/edge/providers/support/orc.py

@@ -23,7 +23,7 @@
 
 
 ORC_APP_LABEL = "app in (aio-orc-api, cert-manager, cainjector, webhook)"
-ORC_CONTROLLER_LABEL = "control-plane in (aio-orc-controller-manager)"
+ORC_CONTROLLER_LABEL = "control-plane in (aio-plat-controller-manager)"
 ORC_DIRECTORY_PATH = ORC_API_V1.moniker
 
 # TODO: @jiacju - this label will be used near future for consistency

azext_edge/edge/providers/support_bundle.py

@@ -15,7 +15,7 @@
 from ..providers.edge_api import (
     CLUSTER_CONFIG_API_V1,
     DATA_PROCESSOR_API_V1,
-    MQ_API_V1B1,
+    MQTT_BROKER_API_V1B1,
     OPCUA_API_V1,
     ORC_API_V1,
     AKRI_API_V0,
@@ -28,7 +28,7 @@
 console = Console()
 
 COMPAT_CLUSTER_CONFIG_APIS = EdgeApiManager(resource_apis=[CLUSTER_CONFIG_API_V1])
-COMPAT_MQ_APIS = EdgeApiManager(resource_apis=[MQ_API_V1B1])
+COMPAT_MQTT_BROKER_APIS = EdgeApiManager(resource_apis=[MQTT_BROKER_API_V1B1])
 COMPAT_OPCUA_APIS = EdgeApiManager(resource_apis=[OPCUA_API_V1])
 COMPAT_DATA_PROCESSOR_APIS = EdgeApiManager(resource_apis=[DATA_PROCESSOR_API_V1])
 COMPAT_ORC_APIS = EdgeApiManager(resource_apis=[ORC_API_V1])
@@ -61,7 +61,7 @@ def build_bundle(
     api_map = {
         # TODO: re-enable billing once service is available post 0.6.0 release
         # OpsServiceType.billing.value: {"apis": COMPAT_CLUSTER_CONFIG_APIS, "prepare_bundle": prepare_billing_bundle},
-        OpsServiceType.mq.value: {"apis": COMPAT_MQ_APIS, "prepare_bundle": prepare_mq_bundle},
+        OpsServiceType.mq.value: {"apis": COMPAT_MQTT_BROKER_APIS, "prepare_bundle": prepare_mq_bundle},
         OpsServiceType.opcua.value: {
             "apis": COMPAT_OPCUA_APIS,
             "prepare_bundle": prepare_opcua_bundle,

azext_edge/tests/edge/checks/conftest.py

@@ -94,7 +94,7 @@ def mock_get_cluster_custom_api(mocker):
 def mock_resource_types(mocker, ops_service):
     patched = mocker.patch("azext_edge.edge.providers.check.base.deployment.enumerate_ops_service_resources")
 
-    if ops_service == "mq":
+    if ops_service == "broker":
         patched.return_value = (
             {},
             {

azext_edge/tests/edge/checks/test_mq_checks_unit.py

@@ -57,7 +57,7 @@
         ],
     ],
 )
-@pytest.mark.parametrize("ops_service", ["mq"])
+@pytest.mark.parametrize("ops_service", ["broker"])
 def test_check_mq_by_resource_types(ops_service, mocker, mock_resource_types, resource_kinds):
     eval_lookup = {
         MqResourceKinds.BROKER.value: "azext_edge.edge.providers.check.mq.evaluate_brokers",

azext_edge/tests/edge/support/conftest.py

@@ -84,8 +84,8 @@ def mocked_cluster_resources(request, mocker):
 
     from azext_edge.edge.providers.edge_api import (
         EdgeResourceApi,
-        MQ_API_V1B1,
         MQ_ACTIVE_API,
+        MQTT_BROKER_API_V1B1,
         OPCUA_API_V1,
         DATA_PROCESSOR_API_V1,
         ORC_API_V1,
@@ -105,7 +105,7 @@ def _get_api_resource(kind: str):
         r_key = r.as_str()
         v1_resources: List[V1APIResource] = []
 
-        if r == MQ_API_V1B1:
+        if r == MQTT_BROKER_API_V1B1:
             v1_resources.append(_get_api_resource("Broker"))
             v1_resources.append(_get_api_resource("BrokerListener"))
             v1_resources.append(_get_api_resource("BrokerDiagnostic"))

azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py

@@ -64,13 +64,13 @@ def test_create_bundle(init_setup, bundle_dir, mq_traces, ops_service, tracked_f
 
     # Check and take out mq traces:
     if mq_traces and ops_service in [OpsServiceType.auto.value, OpsServiceType.mq.value]:
-        mq_level = walk_result.pop(path.join(BASE_ZIP_PATH, namespace, "mq", "traces"), {})
+        mq_level = walk_result.pop(path.join(BASE_ZIP_PATH, namespace, OpsServiceType.mq.value, "traces"), {})
         if mq_level:
             assert not mq_level["folders"]
             assert_file_names(mq_level["files"])
             # make sure level 2 doesnt get messed up
-            assert walk_result[path.join(BASE_ZIP_PATH, namespace, "mq")]["folders"] == ["traces"]
-            walk_result[path.join(BASE_ZIP_PATH, namespace, "mq")]["folders"] = []
+            assert walk_result[path.join(BASE_ZIP_PATH, namespace, OpsServiceType.mq.value)]["folders"] == ["traces"]
+            walk_result[path.join(BASE_ZIP_PATH, namespace, OpsServiceType.mq.value)]["folders"] = []
 
     # Level 2 and 3 - bottom
     actual_walk_result = (len(expected_services) + int("clusterconfig" in expected_services))

azext_edge/tests/edge/support/create_bundle_int/test_mq_int.py

@@ -40,7 +40,7 @@ def test_create_bundle_mq(init_setup, tracked_files, mq_traces):
         resource_api=MQ_ACTIVE_API
     )
 
-    expected_workload_types = ["deployment", "pod", "replicaset", "service", "statefulset"]
+    expected_workload_types = ["pod", "replicaset", "service", "statefulset"]
     expected_types = set(expected_workload_types).union(MQ_ACTIVE_API.kinds)
     assert set(file_map.keys()).issubset(expected_types)