From cc6c3015e27bd7c89822d0211005064a2bb3c899 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Thu, 19 Sep 2024 16:38:02 -0700 Subject: [PATCH 01/19] capture ssc --- azext_edge/edge/common.py | 1 + .../edge/providers/edge_api/__init__.py | 3 + .../edge_api/secretsynccontroller.py | 20 +++++ .../providers/support/secretsynccontroller.py | 77 +++++++++++++++++ azext_edge/edge/providers/support_bundle.py | 8 ++ .../edge/support/create_bundle_int/helpers.py | 11 +++ .../test_secretsynccontroller_int.py | 42 ++++++++++ .../edge/support/test_ssc_support_unit.py | 82 +++++++++++++++++++ 8 files changed, 244 insertions(+) create mode 100644 azext_edge/edge/providers/edge_api/secretsynccontroller.py create mode 100644 azext_edge/edge/providers/support/secretsynccontroller.py create mode 100644 azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py create mode 100644 azext_edge/tests/edge/support/test_ssc_support_unit.py diff --git a/azext_edge/edge/common.py b/azext_edge/edge/common.py index cb9dbb99e..a9999454e 100644 --- a/azext_edge/edge/common.py +++ b/azext_edge/edge/common.py @@ -154,6 +154,7 @@ class OpsServiceType(ListableEnum): dataflow = "dataflow" schemaregistry = "schemaregistry" arccontainerstorage = "acs" + secretsynccontroller = "ssc" @classmethod def list_check_services(cls): diff --git a/azext_edge/edge/providers/edge_api/__init__.py b/azext_edge/edge/providers/edge_api/__init__.py index 1c55d29c8..2d11eb5cf 100644 --- a/azext_edge/edge/providers/edge_api/__init__.py +++ b/azext_edge/edge/providers/edge_api/__init__.py @@ -13,6 +13,7 @@ from .dataflow import DATAFLOW_API_V1B1, DataflowResourceKinds from .meta import META_API_V1B1, MetaResourceKinds from .arccontainerstorage import ARCCONTAINERSTORAGE_API_V1 +from .secretsynccontroller import SECRETSYNC_API_V1, SECRETSTORE_API_V1 __all__ = [ "ARCCONTAINERSTORAGE_API_V1", @@ -32,4 +33,6 @@ "DataflowResourceKinds", "META_API_V1B1", "MetaResourceKinds", + "SECRETSYNC_API_V1", + "SECRETSTORE_API_V1", ] diff --git a/azext_edge/edge/providers/edge_api/secretsynccontroller.py b/azext_edge/edge/providers/edge_api/secretsynccontroller.py new file mode 100644 index 000000000..1b31e29b6 --- /dev/null +++ b/azext_edge/edge/providers/edge_api/secretsynccontroller.py @@ -0,0 +1,20 @@ +# coding=utf-8 +# ---------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License file in the project root for license information. +# ---------------------------------------------------------------------------------------------- + +from .base import EdgeResourceApi + + +SECRETSYNC_API_V1 = EdgeResourceApi( + group="secret-sync.x-k8s.io", + version="v1alpha1", + moniker="secretsync", +) + +SECRETSTORE_API_V1 = EdgeResourceApi( + group="secrets-store.csi.x-k8s.io", + version="v1", + moniker="secretstore", +) diff --git a/azext_edge/edge/providers/support/secretsynccontroller.py b/azext_edge/edge/providers/support/secretsynccontroller.py new file mode 100644 index 000000000..0e3fc818c --- /dev/null +++ b/azext_edge/edge/providers/support/secretsynccontroller.py @@ -0,0 +1,77 @@ +# coding=utf-8 +# ---------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License file in the project root for license information. +# ---------------------------------------------------------------------------------------------- + +from functools import partial +from typing import Iterable, Optional + +from knack.log import get_logger + +from ..edge_api import EdgeResourceApi +from .base import ( + DAY_IN_SECONDS, + assemble_crd_work, + process_deployments, + process_replicasets, + process_services, + process_v1_pods, +) + +logger = get_logger(__name__) + +SSC_DIRECTORY_PATH = "secretsynccontroller" +# TODO: Use common label once it is ready +SSC_NAMESPACE = "azure-secret-store" + + +def fetch_deployments(): + return process_deployments( + directory_path=SSC_DIRECTORY_PATH, + namespace=SSC_NAMESPACE, + ) + + +def fetch_replicasets(): + return process_replicasets( + directory_path=SSC_DIRECTORY_PATH, + namespace=SSC_NAMESPACE, + ) + + +def fetch_pods(since_seconds: int = DAY_IN_SECONDS): + return process_v1_pods( + directory_path=SSC_DIRECTORY_PATH, + since_seconds=since_seconds, + namespace=SSC_NAMESPACE, + ) + + +def fetch_services(): + return process_services( + directory_path=SSC_DIRECTORY_PATH, + namespace=SSC_NAMESPACE, + ) + + +support_runtime_elements = { + "deployments": fetch_deployments, + "replicasets": fetch_replicasets, + "services": fetch_services, +} + + +def prepare_bundle( + log_age_seconds: int = DAY_IN_SECONDS, + apis: Optional[Iterable[EdgeResourceApi]] = None, +) -> dict: + ssc_to_run = {} + + if apis: + ssc_to_run.update(assemble_crd_work(apis=apis, directory_path=SSC_DIRECTORY_PATH)) + + support_runtime_elements["pods"] = partial(fetch_pods, since_seconds=log_age_seconds) + ssc_to_run.update(support_runtime_elements) + + return ssc_to_run diff --git a/azext_edge/edge/providers/support_bundle.py b/azext_edge/edge/providers/support_bundle.py index 988e3af2b..ee34c7d10 100644 --- a/azext_edge/edge/providers/support_bundle.py +++ b/azext_edge/edge/providers/support_bundle.py @@ -20,6 +20,8 @@ DATAFLOW_API_V1B1, META_API_V1B1, ARCCONTAINERSTORAGE_API_V1, + SECRETSYNC_API_V1, + SECRETSTORE_API_V1, EdgeApiManager, ) @@ -34,6 +36,7 @@ COMPAT_DATAFLOW_APIS = EdgeApiManager(resource_apis=[DATAFLOW_API_V1B1]) COMPAT_META_APIS = EdgeApiManager(resource_apis=[META_API_V1B1]) COMPAT_ARCCONTAINERSTORAGE_APIS = EdgeApiManager(resource_apis=[ARCCONTAINERSTORAGE_API_V1]) +COMPAT_SECRETSYNCCONTROLLER_APIS = EdgeApiManager(resource_apis=[SECRETSYNC_API_V1, SECRETSTORE_API_V1]) def build_bundle( @@ -57,6 +60,7 @@ def build_bundle( from .support.meta import prepare_bundle as prepare_meta_bundle from .support.schemaregistry import prepare_bundle as prepare_schema_registry_bundle from .support.arccontainerstorage import prepare_bundle as prepare_arccontainerstorage_bundle + from .support.secretsynccontroller import prepare_bundle as prepare_secretsynccontroller_bundle def collect_default_works( pending_work: dict, @@ -102,6 +106,10 @@ def collect_default_works( "apis": COMPAT_ARCCONTAINERSTORAGE_APIS, "prepare_bundle": prepare_arccontainerstorage_bundle, }, + OpsServiceType.secretsynccontroller.value: { + "apis": COMPAT_SECRETSYNCCONTROLLER_APIS, + "prepare_bundle": prepare_secretsynccontroller_bundle, + }, } for service_moniker, api_info in api_map.items(): diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index 80229e688..4d55b06b5 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -42,6 +42,7 @@ class NamespaceTuple(NamedTuple): arc: str aio: str acs: str + ssc: str usage_system: str @@ -305,6 +306,7 @@ def process_top_levels( clusterconfig_namespace = None arc_namespace = None acs_namespace = None + ssc_namespace = None def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) -> List[str]: level1 = walk_result.get(path.join(BASE_ZIP_PATH, name, folder), {}) @@ -323,6 +325,8 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) arc_namespace = name elif _get_namespace_determinating_files(name=name, folder=path.join("arccontainerstorage"), file_prefix="pvc"): acs_namespace = name + elif _get_namespace_determinating_files(name=name, folder="secretsynccontroller", file_prefix="deployment"): + ssc_namespace = name else: namespace = name @@ -350,11 +354,18 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) assert level_1["folders"] == ["arccontainerstorage"] assert not level_1["files"] + if ssc_namespace: + # remove empty ssc related folders + level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, ssc_namespace)) + assert level_1["folders"] == ["secretsynccontroller"] + assert not level_1["files"] + logger.debug("Determined the following namespaces:") logger.debug(f"AIO namespace: {namespace}") logger.debug(f"Usage system namespace: {clusterconfig_namespace}") logger.debug(f"ARC namespace: {arc_namespace}") logger.debug(f"ACS namespace: {acs_namespace}") + logger.debug(f"SSC namespace: {ssc_namespace}") return NamespaceTuple( arc=arc_namespace, diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py new file mode 100644 index 000000000..6b1383040 --- /dev/null +++ b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py @@ -0,0 +1,42 @@ +# coding=utf-8 +# ---------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License file in the project root for license information. +# ---------------------------------------------------------------------------------------------- + +from knack.log import get_logger +from azext_edge.edge.common import OpsServiceType +from azext_edge.edge.providers.edge_api import SECRETSTORE_API_V1, SECRETSYNC_API_V1 +from .helpers import check_custom_resource_files, check_workload_resource_files, get_file_map, run_bundle_command + +logger = get_logger(__name__) + + +def test_create_bundle_ssc(init_setup, tracked_files): + """Test for ensuring file names and content. ONLY CHECKS arcagents.""" + ops_service = OpsServiceType.secretsynccontroller.value + + command = f"az iot ops support create-bundle --ops-service {ops_service}" + walk_result, bundle_path = run_bundle_command(command=command, tracked_files=tracked_files) + file_map = get_file_map(walk_result, "secretsynccontroller") + + # AIO + check_custom_resource_files( + file_objs=file_map["aio"], resource_api=SECRETSYNC_API_V1, namespace=file_map["__namespaces__"]["aio"] + ) + + # SECRETSTORE + check_custom_resource_files( + file_objs=file_map["secretstore"], + resource_api=SECRETSTORE_API_V1, + namespace=file_map["__namespaces__"]["secretstore"], + ) + expected_workload_types = ["deployment", "pod", "replicaset", "service"] + expected_types = set(expected_workload_types).union(SECRETSTORE_API_V1.kinds) + assert set(file_map["secretstore"].keys()).issubset(expected_types) + check_workload_resource_files( + file_objs=file_map["secretstore"], + expected_workload_types=expected_workload_types, + prefixes=["secrets-store-sync-controller-manager", "manager-metrics-service"], + bundle_path=bundle_path, + ) diff --git a/azext_edge/tests/edge/support/test_ssc_support_unit.py b/azext_edge/tests/edge/support/test_ssc_support_unit.py new file mode 100644 index 000000000..61fa59fab --- /dev/null +++ b/azext_edge/tests/edge/support/test_ssc_support_unit.py @@ -0,0 +1,82 @@ +# coding=utf-8 +# ---------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License file in the project root for license information. +# ---------------------------------------------------------------------------------------------- + +import random + +from azext_edge.edge.commands_edge import support_bundle +from azext_edge.edge.common import OpsServiceType +from azext_edge.edge.providers.support.secretsynccontroller import ( + SSC_DIRECTORY_PATH, + SSC_NAMESPACE, +) +from azext_edge.tests.edge.support.test_support_unit import ( + assert_list_deployments, + assert_list_pods, + assert_list_replica_sets, + assert_list_services, +) + +from ...generators import generate_random_string + +a_bundle_dir = f"support_test_{generate_random_string()}" + + +def test_create_bundle_ssc( + mocked_client, + mocked_config, + mocked_os_makedirs, + mocked_zipfile, + mocked_list_deployments, + mocked_list_pods, + mocked_list_replicasets, + mocked_list_services, + mocked_list_nodes, + mocked_list_cluster_events, + mocked_list_storage_classes, + mocked_root_logger, + mocked_get_config_map, +): + since_seconds = random.randint(86400, 172800) + result = support_bundle( + None, + ops_service=OpsServiceType.secretsynccontroller.value, + bundle_dir=a_bundle_dir, + log_age_seconds=since_seconds, + ) + + assert "bundlePath" in result + assert a_bundle_dir in result["bundlePath"] + + assert_list_pods( + mocked_client, + mocked_zipfile, + mocked_list_pods, + label_selector=None, + directory_path=SSC_DIRECTORY_PATH, + namespace=SSC_NAMESPACE, + since_seconds=since_seconds, + ) + assert_list_deployments( + mocked_client, + mocked_zipfile, + label_selector=None, + directory_path=SSC_DIRECTORY_PATH, + namespace=SSC_NAMESPACE, + ) + assert_list_replica_sets( + mocked_client, + mocked_zipfile, + label_selector=None, + directory_path=SSC_DIRECTORY_PATH, + namespace=SSC_NAMESPACE, + ) + assert_list_services( + mocked_client, + mocked_zipfile, + label_selector=None, + directory_path=SSC_DIRECTORY_PATH, + namespace=SSC_NAMESPACE, + ) From b60dab8b43a87bd69b403a89e4975c90feba3fa5 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Thu, 19 Sep 2024 17:06:13 -0700 Subject: [PATCH 02/19] update --- .../tests/edge/support/create_bundle_int/helpers.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index 4d55b06b5..c88501061 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -239,7 +239,9 @@ def get_file_map( mq_traces: bool = False, ) -> Dict[str, Dict[str, List[Dict[str, str]]]]: # Remove all files that will not be checked - arc_namespace, aio_namespace, acs_namespace, c_namespace = process_top_levels(walk_result, ops_service) + arc_namespace, aio_namespace, acs_namespace, ssc_namespace, c_namespace = process_top_levels( + walk_result, ops_service + ) if aio_namespace: walk_result.pop(path.join(BASE_ZIP_PATH, aio_namespace)) @@ -274,9 +276,11 @@ def get_file_map( acs_path = path.join(BASE_ZIP_PATH, acs_namespace, "arccontainerstorage") file_map["acs"] = convert_file_names(walk_result[acs_path]["files"]) file_map["__namespaces__"]["acs"] = acs_namespace - - # no files for aio, skip the rest assertions - return file_map + elif ops_service == "ssc": + assert len(walk_result) == 1 + expected_default_walk_result + ssc_path = path.join(BASE_ZIP_PATH, ssc_namespace, "secretsynccontroller") + file_map["ssc"] = convert_file_names(walk_result[ssc_path]["files"]) + file_map["__namespaces__"]["ssc"] = ssc_namespace elif ops_service == "deviceregistry": if ops_path not in walk_result: assert len(walk_result) == expected_default_walk_result @@ -371,6 +375,7 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) arc=arc_namespace, aio=namespace, acs=acs_namespace, + ssc=ssc_namespace, usage_system=clusterconfig_namespace, ) From 460d28565ab7e1e57bfa249adef78f0c2abb4898 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Thu, 19 Sep 2024 17:14:18 -0700 Subject: [PATCH 03/19] update2 --- .../support/create_bundle_int/test_secretsynccontroller_int.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py index 6b1383040..9ab3cb441 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py +++ b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py @@ -18,7 +18,7 @@ def test_create_bundle_ssc(init_setup, tracked_files): command = f"az iot ops support create-bundle --ops-service {ops_service}" walk_result, bundle_path = run_bundle_command(command=command, tracked_files=tracked_files) - file_map = get_file_map(walk_result, "secretsynccontroller") + file_map = get_file_map(walk_result, ops_service) # AIO check_custom_resource_files( From 35f797ceefa3dd1fccbe7bccfcb34439fb170c20 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Thu, 19 Sep 2024 17:47:15 -0700 Subject: [PATCH 04/19] update3 --- .../tests/edge/support/create_bundle_int/helpers.py | 3 ++- .../create_bundle_int/test_secretsynccontroller_int.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index c88501061..793620741 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -277,7 +277,8 @@ def get_file_map( file_map["acs"] = convert_file_names(walk_result[acs_path]["files"]) file_map["__namespaces__"]["acs"] = acs_namespace elif ops_service == "ssc": - assert len(walk_result) == 1 + expected_default_walk_result + assert len(walk_result) == 2 + expected_default_walk_result + ops_path = path.join(BASE_ZIP_PATH, aio_namespace, "secretsynccontroller") ssc_path = path.join(BASE_ZIP_PATH, ssc_namespace, "secretsynccontroller") file_map["ssc"] = convert_file_names(walk_result[ssc_path]["files"]) file_map["__namespaces__"]["ssc"] = ssc_namespace diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py index 9ab3cb441..ed8b1bd38 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py +++ b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py @@ -27,15 +27,15 @@ def test_create_bundle_ssc(init_setup, tracked_files): # SECRETSTORE check_custom_resource_files( - file_objs=file_map["secretstore"], + file_objs=file_map["ssc"], resource_api=SECRETSTORE_API_V1, - namespace=file_map["__namespaces__"]["secretstore"], + namespace=file_map["__namespaces__"]["ssc"], ) expected_workload_types = ["deployment", "pod", "replicaset", "service"] expected_types = set(expected_workload_types).union(SECRETSTORE_API_V1.kinds) - assert set(file_map["secretstore"].keys()).issubset(expected_types) + assert set(file_map["ssc"].keys()).issubset(expected_types) check_workload_resource_files( - file_objs=file_map["secretstore"], + file_objs=file_map["ssc"], expected_workload_types=expected_workload_types, prefixes=["secrets-store-sync-controller-manager", "manager-metrics-service"], bundle_path=bundle_path, From b9f51bab340a3ffdd40caed855fe453982c142c7 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 20 Sep 2024 12:28:13 -0700 Subject: [PATCH 05/19] update4 --- .../create_bundle_int/test_auto_int.py | 19 ++++++++++++++++++- .../test_secretsynccontroller_int.py | 10 +++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py index 0d3da8a58..14730ccac 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py +++ b/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py @@ -77,6 +77,10 @@ def test_create_bundle(init_setup, bundle_dir, mq_traces, ops_service, tracked_f if namespace.acs: walk_result.pop(path.join(BASE_ZIP_PATH, namespace.acs, "arccontainerstorage"), {}) + # remove ssc resources in ssc namespace from walk_result from aio namespace assertion + if namespace.ssc: + walk_result.pop(path.join(BASE_ZIP_PATH, namespace.ssc, "secretsynccontroller"), {}) + # Level 2 and 3 - bottom is_billing_included = OpsServiceType.billing.value in expected_services actual_walk_result = len(expected_services) + int(is_billing_included) + len(ARC_AGENTS) @@ -109,11 +113,17 @@ def test_create_bundle(init_setup, bundle_dir, mq_traces, ops_service, tracked_f def _get_expected_services( walk_result: Dict[str, Dict[str, List[str]]], ops_service: str, namespace: str ) -> List[str]: - expected_services = [ops_service] + expected_services = ( + [ops_service] if ops_service != OpsServiceType.secretsynccontroller.value else ["secretsynccontroller"] + ) if ops_service == OpsServiceType.auto.value: # these should always be generated expected_services = OpsServiceType.list() expected_services.remove(OpsServiceType.auto.value) + + # folder name for secretsynccontroller is different from the service name + expected_services.remove(OpsServiceType.secretsynccontroller.value) + expected_services.append("secretsynccontroller") expected_services.sort() # device registry folder will not be created if there are no device registry resources @@ -130,5 +140,12 @@ def _get_expected_services( ): expected_services.remove(OpsServiceType.arccontainerstorage.value) + # secretsynccontroller folder will not be created if there are no secretsynccontroller resources + if ( + not walk_result.get(path.join(BASE_ZIP_PATH, namespace, "secretsynccontroller")) + and "secretsynccontroller" in expected_services + ): + expected_services.remove("secretsynccontroller") + expected_services.append("meta") return expected_services diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py index ed8b1bd38..1d4da0eeb 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py +++ b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py @@ -24,15 +24,15 @@ def test_create_bundle_ssc(init_setup, tracked_files): check_custom_resource_files( file_objs=file_map["aio"], resource_api=SECRETSYNC_API_V1, namespace=file_map["__namespaces__"]["aio"] ) - - # SECRETSTORE check_custom_resource_files( - file_objs=file_map["ssc"], + file_objs=file_map["aio"], resource_api=SECRETSTORE_API_V1, - namespace=file_map["__namespaces__"]["ssc"], + namespace=file_map["__namespaces__"]["aio"], ) + + # SECRETSTORE expected_workload_types = ["deployment", "pod", "replicaset", "service"] - expected_types = set(expected_workload_types).union(SECRETSTORE_API_V1.kinds) + expected_types = set(expected_workload_types) assert set(file_map["ssc"].keys()).issubset(expected_types) check_workload_resource_files( file_objs=file_map["ssc"], From 7ee6ff977117e7b83dd4d30d52666d08badcbe84 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 20 Sep 2024 12:53:13 -0700 Subject: [PATCH 06/19] update5 --- azext_edge/tests/edge/support/create_bundle_int/helpers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index 793620741..eed41463b 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -276,6 +276,9 @@ def get_file_map( acs_path = path.join(BASE_ZIP_PATH, acs_namespace, "arccontainerstorage") file_map["acs"] = convert_file_names(walk_result[acs_path]["files"]) file_map["__namespaces__"]["acs"] = acs_namespace + + # no files for aio, skip the rest assertions + return file_map elif ops_service == "ssc": assert len(walk_result) == 2 + expected_default_walk_result ops_path = path.join(BASE_ZIP_PATH, aio_namespace, "secretsynccontroller") From 6208683126e6bf2b0e88cbe3489cd1a530375a75 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 20 Sep 2024 14:11:38 -0700 Subject: [PATCH 07/19] update6 --- azext_edge/tests/edge/support/create_bundle_int/helpers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index eed41463b..7a9e0ece7 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -280,9 +280,13 @@ def get_file_map( # no files for aio, skip the rest assertions return file_map elif ops_service == "ssc": - assert len(walk_result) == 2 + expected_default_walk_result ops_path = path.join(BASE_ZIP_PATH, aio_namespace, "secretsynccontroller") ssc_path = path.join(BASE_ZIP_PATH, ssc_namespace, "secretsynccontroller") + if ops_path not in walk_result: + # no crd created in aio namespace + assert len(walk_result) == 1 + expected_default_walk_result + else: + assert len(walk_result) == 2 + expected_default_walk_result file_map["ssc"] = convert_file_names(walk_result[ssc_path]["files"]) file_map["__namespaces__"]["ssc"] = ssc_namespace elif ops_service == "deviceregistry": From e997e39cc06785d470044fb090937a8a6a5797dd Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 20 Sep 2024 14:25:11 -0700 Subject: [PATCH 08/19] update7 --- .../edge/support/create_bundle_int/helpers.py | 65 +++++++++++++------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index 7a9e0ece7..d4f8cc25e 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -342,36 +342,59 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) else: namespace = name + # if clusterconfig_namespace: + # # remove empty billing related folders + # level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, clusterconfig_namespace)) + # assert level_1["folders"] == ["clusterconfig"] + # assert not level_1["files"] + # level_2 = walk_result.pop(path.join(BASE_ZIP_PATH, clusterconfig_namespace, "clusterconfig")) + # assert level_2["folders"] == ["billing"] + # assert not level_2["files"] + + # if arc_namespace: + # # remove empty arc related folders + # level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, arc_namespace)) + # assert level_1["folders"] == ["arcagents"] + # assert not level_1["files"] + # level_2 = walk_result.pop(path.join(BASE_ZIP_PATH, arc_namespace, "arcagents")) + # assert level_2["folders"] == [agent[0] for agent in ARC_AGENTS] + # assert not level_2["files"] + + # if acs_namespace: + # # remove empty acs related folders + # level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, acs_namespace)) + # assert level_1["folders"] == ["arccontainerstorage"] + # assert not level_1["files"] + + # if ssc_namespace: + # # remove empty ssc related folders + # level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, ssc_namespace)) + # assert level_1["folders"] == ["secretsynccontroller"] + # assert not level_1["files"] + + for namespace, folder in [ + (namespace, ops_service), + (clusterconfig_namespace, "clusterconfig"), + (arc_namespace, ARC_AGENTS[0][0]), + (acs_namespace, "arccontainerstorage"), + (ssc_namespace, "secretsynccontroller"), + ]: + if namespace: + # remove empty folders in level 1 + level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, namespace)) + assert level_1["folders"] == [folder] + assert not level_1["files"] + + # remove empty folders in level 2 if clusterconfig_namespace: - # remove empty billing related folders - level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, clusterconfig_namespace)) - assert level_1["folders"] == ["clusterconfig"] - assert not level_1["files"] level_2 = walk_result.pop(path.join(BASE_ZIP_PATH, clusterconfig_namespace, "clusterconfig")) assert level_2["folders"] == ["billing"] assert not level_2["files"] - if arc_namespace: - # remove empty arc related folders - level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, arc_namespace)) - assert level_1["folders"] == ["arcagents"] - assert not level_1["files"] level_2 = walk_result.pop(path.join(BASE_ZIP_PATH, arc_namespace, "arcagents")) assert level_2["folders"] == [agent[0] for agent in ARC_AGENTS] assert not level_2["files"] - if acs_namespace: - # remove empty acs related folders - level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, acs_namespace)) - assert level_1["folders"] == ["arccontainerstorage"] - assert not level_1["files"] - - if ssc_namespace: - # remove empty ssc related folders - level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, ssc_namespace)) - assert level_1["folders"] == ["secretsynccontroller"] - assert not level_1["files"] - logger.debug("Determined the following namespaces:") logger.debug(f"AIO namespace: {namespace}") logger.debug(f"Usage system namespace: {clusterconfig_namespace}") From d3236ad41304993bf63296ad65487f358928dfc2 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 20 Sep 2024 14:46:18 -0700 Subject: [PATCH 09/19] update8 --- azext_edge/tests/edge/support/create_bundle_int/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index d4f8cc25e..4c50ec77a 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -375,7 +375,7 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) for namespace, folder in [ (namespace, ops_service), (clusterconfig_namespace, "clusterconfig"), - (arc_namespace, ARC_AGENTS[0][0]), + (arc_namespace, "arcagents"), (acs_namespace, "arccontainerstorage"), (ssc_namespace, "secretsynccontroller"), ]: From 38323ddebd4b809ec485581665a3b3aa664cb5ff Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 20 Sep 2024 15:10:57 -0700 Subject: [PATCH 10/19] update9 --- azext_edge/tests/edge/support/create_bundle_int/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index 4c50ec77a..e255a33b1 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -373,7 +373,6 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) # assert not level_1["files"] for namespace, folder in [ - (namespace, ops_service), (clusterconfig_namespace, "clusterconfig"), (arc_namespace, "arcagents"), (acs_namespace, "arccontainerstorage"), From a4f34eba145ad2d7b8e1437bb04507e7e2dbea6d Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 20 Sep 2024 15:24:26 -0700 Subject: [PATCH 11/19] update10 --- .../tests/edge/support/create_bundle_int/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index e255a33b1..cd26d8b8e 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -372,16 +372,16 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) # assert level_1["folders"] == ["secretsynccontroller"] # assert not level_1["files"] - for namespace, folder in [ + for namespace_folder, service in [ (clusterconfig_namespace, "clusterconfig"), (arc_namespace, "arcagents"), (acs_namespace, "arccontainerstorage"), (ssc_namespace, "secretsynccontroller"), ]: - if namespace: + if namespace_folder: # remove empty folders in level 1 - level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, namespace)) - assert level_1["folders"] == [folder] + level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, namespace_folder)) + assert level_1["folders"] == [service] assert not level_1["files"] # remove empty folders in level 2 From 22da42f65abf040e0bd3cfbc52eb2db50af6e529 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 20 Sep 2024 17:10:05 -0700 Subject: [PATCH 12/19] update11 --- .../edge/support/create_bundle_int/helpers.py | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index cd26d8b8e..c6139d0df 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -342,36 +342,6 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) else: namespace = name - # if clusterconfig_namespace: - # # remove empty billing related folders - # level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, clusterconfig_namespace)) - # assert level_1["folders"] == ["clusterconfig"] - # assert not level_1["files"] - # level_2 = walk_result.pop(path.join(BASE_ZIP_PATH, clusterconfig_namespace, "clusterconfig")) - # assert level_2["folders"] == ["billing"] - # assert not level_2["files"] - - # if arc_namespace: - # # remove empty arc related folders - # level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, arc_namespace)) - # assert level_1["folders"] == ["arcagents"] - # assert not level_1["files"] - # level_2 = walk_result.pop(path.join(BASE_ZIP_PATH, arc_namespace, "arcagents")) - # assert level_2["folders"] == [agent[0] for agent in ARC_AGENTS] - # assert not level_2["files"] - - # if acs_namespace: - # # remove empty acs related folders - # level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, acs_namespace)) - # assert level_1["folders"] == ["arccontainerstorage"] - # assert not level_1["files"] - - # if ssc_namespace: - # # remove empty ssc related folders - # level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, ssc_namespace)) - # assert level_1["folders"] == ["secretsynccontroller"] - # assert not level_1["files"] - for namespace_folder, service in [ (clusterconfig_namespace, "clusterconfig"), (arc_namespace, "arcagents"), From 3b5656b089d3520d1d5dcc02669608114902aaac Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 20 Sep 2024 17:26:27 -0700 Subject: [PATCH 13/19] rename to secret store --- azext_edge/edge/common.py | 2 +- .../providers/support/secretsynccontroller.py | 2 +- .../edge/support/create_bundle_int/helpers.py | 17 ++++++++++------- .../support/create_bundle_int/test_auto_int.py | 18 ++++++------------ .../test_secretsynccontroller_int.py | 4 ++-- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/azext_edge/edge/common.py b/azext_edge/edge/common.py index a9999454e..f767438ec 100644 --- a/azext_edge/edge/common.py +++ b/azext_edge/edge/common.py @@ -154,7 +154,7 @@ class OpsServiceType(ListableEnum): dataflow = "dataflow" schemaregistry = "schemaregistry" arccontainerstorage = "acs" - secretsynccontroller = "ssc" + secretsynccontroller = "secretstore" @classmethod def list_check_services(cls): diff --git a/azext_edge/edge/providers/support/secretsynccontroller.py b/azext_edge/edge/providers/support/secretsynccontroller.py index 0e3fc818c..0f5f885a3 100644 --- a/azext_edge/edge/providers/support/secretsynccontroller.py +++ b/azext_edge/edge/providers/support/secretsynccontroller.py @@ -21,7 +21,7 @@ logger = get_logger(__name__) -SSC_DIRECTORY_PATH = "secretsynccontroller" +SSC_DIRECTORY_PATH = "secretstore" # TODO: Use common label once it is ready SSC_NAMESPACE = "azure-secret-store" diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index c6139d0df..7eb40bd45 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -10,6 +10,7 @@ from zipfile import ZipFile import pytest from azure.cli.core.azclierror import CLIInternalError +from azext_edge.edge.common import OpsServiceType from azext_edge.edge.providers.edge_api.base import EdgeResourceApi from azext_edge.edge.providers.support.arcagents import ARC_AGENTS from ....helpers import ( @@ -279,16 +280,16 @@ def get_file_map( # no files for aio, skip the rest assertions return file_map - elif ops_service == "ssc": - ops_path = path.join(BASE_ZIP_PATH, aio_namespace, "secretsynccontroller") - ssc_path = path.join(BASE_ZIP_PATH, ssc_namespace, "secretsynccontroller") + elif ops_service == OpsServiceType.secretsynccontroller.value: + ops_path = path.join(BASE_ZIP_PATH, aio_namespace, OpsServiceType.secretsynccontroller.value) + ssc_path = path.join(BASE_ZIP_PATH, ssc_namespace, OpsServiceType.secretsynccontroller.value) if ops_path not in walk_result: # no crd created in aio namespace assert len(walk_result) == 1 + expected_default_walk_result else: assert len(walk_result) == 2 + expected_default_walk_result - file_map["ssc"] = convert_file_names(walk_result[ssc_path]["files"]) - file_map["__namespaces__"]["ssc"] = ssc_namespace + file_map[OpsServiceType.secretsynccontroller.value] = convert_file_names(walk_result[ssc_path]["files"]) + file_map["__namespaces__"][OpsServiceType.secretsynccontroller.value] = ssc_namespace elif ops_service == "deviceregistry": if ops_path not in walk_result: assert len(walk_result) == expected_default_walk_result @@ -337,7 +338,9 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) arc_namespace = name elif _get_namespace_determinating_files(name=name, folder=path.join("arccontainerstorage"), file_prefix="pvc"): acs_namespace = name - elif _get_namespace_determinating_files(name=name, folder="secretsynccontroller", file_prefix="deployment"): + elif _get_namespace_determinating_files( + name=name, folder=OpsServiceType.secretsynccontroller.value, file_prefix="deployment" + ): ssc_namespace = name else: namespace = name @@ -346,7 +349,7 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) (clusterconfig_namespace, "clusterconfig"), (arc_namespace, "arcagents"), (acs_namespace, "arccontainerstorage"), - (ssc_namespace, "secretsynccontroller"), + (ssc_namespace, OpsServiceType.secretsynccontroller.value), ]: if namespace_folder: # remove empty folders in level 1 diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py index 14730ccac..3852cd53c 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py +++ b/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py @@ -79,7 +79,7 @@ def test_create_bundle(init_setup, bundle_dir, mq_traces, ops_service, tracked_f # remove ssc resources in ssc namespace from walk_result from aio namespace assertion if namespace.ssc: - walk_result.pop(path.join(BASE_ZIP_PATH, namespace.ssc, "secretsynccontroller"), {}) + walk_result.pop(path.join(BASE_ZIP_PATH, namespace.ssc, OpsServiceType.secretsynccontroller.value), {}) # Level 2 and 3 - bottom is_billing_included = OpsServiceType.billing.value in expected_services @@ -113,17 +113,11 @@ def test_create_bundle(init_setup, bundle_dir, mq_traces, ops_service, tracked_f def _get_expected_services( walk_result: Dict[str, Dict[str, List[str]]], ops_service: str, namespace: str ) -> List[str]: - expected_services = ( - [ops_service] if ops_service != OpsServiceType.secretsynccontroller.value else ["secretsynccontroller"] - ) + expected_services = [ops_service] if ops_service == OpsServiceType.auto.value: # these should always be generated expected_services = OpsServiceType.list() expected_services.remove(OpsServiceType.auto.value) - - # folder name for secretsynccontroller is different from the service name - expected_services.remove(OpsServiceType.secretsynccontroller.value) - expected_services.append("secretsynccontroller") expected_services.sort() # device registry folder will not be created if there are no device registry resources @@ -140,12 +134,12 @@ def _get_expected_services( ): expected_services.remove(OpsServiceType.arccontainerstorage.value) - # secretsynccontroller folder will not be created if there are no secretsynccontroller resources + # secretstore folder will not be created if there are no secretstore resources if ( - not walk_result.get(path.join(BASE_ZIP_PATH, namespace, "secretsynccontroller")) - and "secretsynccontroller" in expected_services + not walk_result.get(path.join(BASE_ZIP_PATH, namespace, OpsServiceType.secretsynccontroller.value)) + and OpsServiceType.secretsynccontroller.value in expected_services ): - expected_services.remove("secretsynccontroller") + expected_services.remove(OpsServiceType.secretsynccontroller.value) expected_services.append("meta") return expected_services diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py index 1d4da0eeb..8fca62a61 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py +++ b/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py @@ -33,9 +33,9 @@ def test_create_bundle_ssc(init_setup, tracked_files): # SECRETSTORE expected_workload_types = ["deployment", "pod", "replicaset", "service"] expected_types = set(expected_workload_types) - assert set(file_map["ssc"].keys()).issubset(expected_types) + assert set(file_map[OpsServiceType.secretsynccontroller.value].keys()).issubset(expected_types) check_workload_resource_files( - file_objs=file_map["ssc"], + file_objs=file_map[OpsServiceType.secretsynccontroller.value], expected_workload_types=expected_workload_types, prefixes=["secrets-store-sync-controller-manager", "manager-metrics-service"], bundle_path=bundle_path, From d2043805ccd72475dab47f8c81afcf4ffcc7a866 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Mon, 23 Sep 2024 11:07:40 -0700 Subject: [PATCH 14/19] update help --- azext_edge/edge/_help.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/azext_edge/edge/_help.py b/azext_edge/edge/_help.py index 24a1ce86f..c98fe1378 100644 --- a/azext_edge/edge/_help.py +++ b/azext_edge/edge/_help.py @@ -17,6 +17,7 @@ COMPAT_MQTT_BROKER_APIS, COMPAT_OPCUA_APIS, COMPAT_DATAFLOW_APIS, + COMPAT_SECRETSYNCCONTROLLER_APIS, ) @@ -56,6 +57,7 @@ def load_iotops_help(): - {COMPAT_CLUSTER_CONFIG_APIS.as_str()} - {COMPAT_DATAFLOW_APIS.as_str()} - {COMPAT_ARCCONTAINERSTORAGE_APIS.as_str()} + - {COMPAT_SECRETSYNCCONTROLLER_APIS.as_str()} Note: logs from evicted pod will not be captured, as they are inaccessible. For details on why a pod was evicted, please refer to the related pod and node files. @@ -81,6 +83,10 @@ def load_iotops_help(): - name: Include arc container storage resources in the support bundle. text: > az iot ops support create-bundle --ops-service acs + + - name: Include secretstore resources in the support bundle. + text: > + az iot ops support create-bundle --ops-service secretstore """ helps[ From 06d65275b83b48d0662e491b632dbe70d3ccc54c Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Mon, 23 Sep 2024 12:15:57 -0700 Subject: [PATCH 15/19] update help2 --- azext_edge/edge/_help.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azext_edge/edge/_help.py b/azext_edge/edge/_help.py index c98fe1378..c315b2a18 100644 --- a/azext_edge/edge/_help.py +++ b/azext_edge/edge/_help.py @@ -9,6 +9,8 @@ from knack.help_files import helps +from azext_edge.edge.providers.edge_api import SECRETSTORE_API_V1, SECRETSYNC_API_V1 + from .providers.edge_api import MQ_ACTIVE_API from .providers.support_bundle import ( COMPAT_ARCCONTAINERSTORAGE_APIS, @@ -17,7 +19,6 @@ COMPAT_MQTT_BROKER_APIS, COMPAT_OPCUA_APIS, COMPAT_DATAFLOW_APIS, - COMPAT_SECRETSYNCCONTROLLER_APIS, ) @@ -57,7 +58,8 @@ def load_iotops_help(): - {COMPAT_CLUSTER_CONFIG_APIS.as_str()} - {COMPAT_DATAFLOW_APIS.as_str()} - {COMPAT_ARCCONTAINERSTORAGE_APIS.as_str()} - - {COMPAT_SECRETSYNCCONTROLLER_APIS.as_str()} + - {SECRETSYNC_API_V1.as_str()} + - {SECRETSTORE_API_V1.as_str()} Note: logs from evicted pod will not be captured, as they are inaccessible. For details on why a pod was evicted, please refer to the related pod and node files. @@ -83,7 +85,7 @@ def load_iotops_help(): - name: Include arc container storage resources in the support bundle. text: > az iot ops support create-bundle --ops-service acs - + - name: Include secretstore resources in the support bundle. text: > az iot ops support create-bundle --ops-service secretstore From a393fb55c45ded2f6cf7de937754bf5b9c004e73 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Mon, 23 Sep 2024 13:01:49 -0700 Subject: [PATCH 16/19] further rename --- azext_edge/edge/common.py | 2 +- azext_edge/edge/providers/edge_api/__init__.py | 2 +- .../{secretsynccontroller.py => secretstore.py} | 0 .../{secretsynccontroller.py => secretstore.py} | 0 azext_edge/edge/providers/support_bundle.py | 10 +++++----- .../edge/support/create_bundle_int/helpers.py | 14 +++++++------- .../support/create_bundle_int/test_auto_int.py | 8 ++++---- ...nccontroller_int.py => test_secretstore_int.py} | 0 ...rt_unit.py => test_secretstore_support_unit.py} | 4 ++-- 9 files changed, 20 insertions(+), 20 deletions(-) rename azext_edge/edge/providers/edge_api/{secretsynccontroller.py => secretstore.py} (100%) rename azext_edge/edge/providers/support/{secretsynccontroller.py => secretstore.py} (100%) rename azext_edge/tests/edge/support/create_bundle_int/{test_secretsynccontroller_int.py => test_secretstore_int.py} (100%) rename azext_edge/tests/edge/support/{test_ssc_support_unit.py => test_secretstore_support_unit.py} (94%) diff --git a/azext_edge/edge/common.py b/azext_edge/edge/common.py index f767438ec..9cdce19ac 100644 --- a/azext_edge/edge/common.py +++ b/azext_edge/edge/common.py @@ -154,7 +154,7 @@ class OpsServiceType(ListableEnum): dataflow = "dataflow" schemaregistry = "schemaregistry" arccontainerstorage = "acs" - secretsynccontroller = "secretstore" + secretstore = "secretstore" @classmethod def list_check_services(cls): diff --git a/azext_edge/edge/providers/edge_api/__init__.py b/azext_edge/edge/providers/edge_api/__init__.py index 2d11eb5cf..44d1f9cb2 100644 --- a/azext_edge/edge/providers/edge_api/__init__.py +++ b/azext_edge/edge/providers/edge_api/__init__.py @@ -13,7 +13,7 @@ from .dataflow import DATAFLOW_API_V1B1, DataflowResourceKinds from .meta import META_API_V1B1, MetaResourceKinds from .arccontainerstorage import ARCCONTAINERSTORAGE_API_V1 -from .secretsynccontroller import SECRETSYNC_API_V1, SECRETSTORE_API_V1 +from .secretstore import SECRETSYNC_API_V1, SECRETSTORE_API_V1 __all__ = [ "ARCCONTAINERSTORAGE_API_V1", diff --git a/azext_edge/edge/providers/edge_api/secretsynccontroller.py b/azext_edge/edge/providers/edge_api/secretstore.py similarity index 100% rename from azext_edge/edge/providers/edge_api/secretsynccontroller.py rename to azext_edge/edge/providers/edge_api/secretstore.py diff --git a/azext_edge/edge/providers/support/secretsynccontroller.py b/azext_edge/edge/providers/support/secretstore.py similarity index 100% rename from azext_edge/edge/providers/support/secretsynccontroller.py rename to azext_edge/edge/providers/support/secretstore.py diff --git a/azext_edge/edge/providers/support_bundle.py b/azext_edge/edge/providers/support_bundle.py index ee34c7d10..55c1fa5a0 100644 --- a/azext_edge/edge/providers/support_bundle.py +++ b/azext_edge/edge/providers/support_bundle.py @@ -36,7 +36,7 @@ COMPAT_DATAFLOW_APIS = EdgeApiManager(resource_apis=[DATAFLOW_API_V1B1]) COMPAT_META_APIS = EdgeApiManager(resource_apis=[META_API_V1B1]) COMPAT_ARCCONTAINERSTORAGE_APIS = EdgeApiManager(resource_apis=[ARCCONTAINERSTORAGE_API_V1]) -COMPAT_SECRETSYNCCONTROLLER_APIS = EdgeApiManager(resource_apis=[SECRETSYNC_API_V1, SECRETSTORE_API_V1]) +COMPAT_SECRETSTORE_APIS = EdgeApiManager(resource_apis=[SECRETSYNC_API_V1, SECRETSTORE_API_V1]) def build_bundle( @@ -60,7 +60,7 @@ def build_bundle( from .support.meta import prepare_bundle as prepare_meta_bundle from .support.schemaregistry import prepare_bundle as prepare_schema_registry_bundle from .support.arccontainerstorage import prepare_bundle as prepare_arccontainerstorage_bundle - from .support.secretsynccontroller import prepare_bundle as prepare_secretsynccontroller_bundle + from .support.secretstore import prepare_bundle as prepare_secretstore_bundle def collect_default_works( pending_work: dict, @@ -106,9 +106,9 @@ def collect_default_works( "apis": COMPAT_ARCCONTAINERSTORAGE_APIS, "prepare_bundle": prepare_arccontainerstorage_bundle, }, - OpsServiceType.secretsynccontroller.value: { - "apis": COMPAT_SECRETSYNCCONTROLLER_APIS, - "prepare_bundle": prepare_secretsynccontroller_bundle, + OpsServiceType.secretstore.value: { + "apis": COMPAT_SECRETSTORE_APIS, + "prepare_bundle": prepare_secretstore_bundle, }, } diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index 7eb40bd45..4392c2d9b 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -280,16 +280,16 @@ def get_file_map( # no files for aio, skip the rest assertions return file_map - elif ops_service == OpsServiceType.secretsynccontroller.value: - ops_path = path.join(BASE_ZIP_PATH, aio_namespace, OpsServiceType.secretsynccontroller.value) - ssc_path = path.join(BASE_ZIP_PATH, ssc_namespace, OpsServiceType.secretsynccontroller.value) + elif ops_service == OpsServiceType.secretstore.value: + ops_path = path.join(BASE_ZIP_PATH, aio_namespace, OpsServiceType.secretstore.value) + ssc_path = path.join(BASE_ZIP_PATH, ssc_namespace, OpsServiceType.secretstore.value) if ops_path not in walk_result: # no crd created in aio namespace assert len(walk_result) == 1 + expected_default_walk_result else: assert len(walk_result) == 2 + expected_default_walk_result - file_map[OpsServiceType.secretsynccontroller.value] = convert_file_names(walk_result[ssc_path]["files"]) - file_map["__namespaces__"][OpsServiceType.secretsynccontroller.value] = ssc_namespace + file_map[OpsServiceType.secretstore.value] = convert_file_names(walk_result[ssc_path]["files"]) + file_map["__namespaces__"][OpsServiceType.secretstore.value] = ssc_namespace elif ops_service == "deviceregistry": if ops_path not in walk_result: assert len(walk_result) == expected_default_walk_result @@ -339,7 +339,7 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) elif _get_namespace_determinating_files(name=name, folder=path.join("arccontainerstorage"), file_prefix="pvc"): acs_namespace = name elif _get_namespace_determinating_files( - name=name, folder=OpsServiceType.secretsynccontroller.value, file_prefix="deployment" + name=name, folder=OpsServiceType.secretstore.value, file_prefix="deployment" ): ssc_namespace = name else: @@ -349,7 +349,7 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) (clusterconfig_namespace, "clusterconfig"), (arc_namespace, "arcagents"), (acs_namespace, "arccontainerstorage"), - (ssc_namespace, OpsServiceType.secretsynccontroller.value), + (ssc_namespace, OpsServiceType.secretstore.value), ]: if namespace_folder: # remove empty folders in level 1 diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py index 3852cd53c..04cd9bdaf 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py +++ b/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py @@ -79,7 +79,7 @@ def test_create_bundle(init_setup, bundle_dir, mq_traces, ops_service, tracked_f # remove ssc resources in ssc namespace from walk_result from aio namespace assertion if namespace.ssc: - walk_result.pop(path.join(BASE_ZIP_PATH, namespace.ssc, OpsServiceType.secretsynccontroller.value), {}) + walk_result.pop(path.join(BASE_ZIP_PATH, namespace.ssc, OpsServiceType.secretstore.value), {}) # Level 2 and 3 - bottom is_billing_included = OpsServiceType.billing.value in expected_services @@ -136,10 +136,10 @@ def _get_expected_services( # secretstore folder will not be created if there are no secretstore resources if ( - not walk_result.get(path.join(BASE_ZIP_PATH, namespace, OpsServiceType.secretsynccontroller.value)) - and OpsServiceType.secretsynccontroller.value in expected_services + not walk_result.get(path.join(BASE_ZIP_PATH, namespace, OpsServiceType.secretstore.value)) + and OpsServiceType.secretstore.value in expected_services ): - expected_services.remove(OpsServiceType.secretsynccontroller.value) + expected_services.remove(OpsServiceType.secretstore.value) expected_services.append("meta") return expected_services diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_secretstore_int.py similarity index 100% rename from azext_edge/tests/edge/support/create_bundle_int/test_secretsynccontroller_int.py rename to azext_edge/tests/edge/support/create_bundle_int/test_secretstore_int.py diff --git a/azext_edge/tests/edge/support/test_ssc_support_unit.py b/azext_edge/tests/edge/support/test_secretstore_support_unit.py similarity index 94% rename from azext_edge/tests/edge/support/test_ssc_support_unit.py rename to azext_edge/tests/edge/support/test_secretstore_support_unit.py index 61fa59fab..bf9d65304 100644 --- a/azext_edge/tests/edge/support/test_ssc_support_unit.py +++ b/azext_edge/tests/edge/support/test_secretstore_support_unit.py @@ -8,7 +8,7 @@ from azext_edge.edge.commands_edge import support_bundle from azext_edge.edge.common import OpsServiceType -from azext_edge.edge.providers.support.secretsynccontroller import ( +from azext_edge.edge.providers.support.secretstore import ( SSC_DIRECTORY_PATH, SSC_NAMESPACE, ) @@ -42,7 +42,7 @@ def test_create_bundle_ssc( since_seconds = random.randint(86400, 172800) result = support_bundle( None, - ops_service=OpsServiceType.secretsynccontroller.value, + ops_service=OpsServiceType.secretstore.value, bundle_dir=a_bundle_dir, log_age_seconds=since_seconds, ) From f985fe71c48ca445397e92c163719aff2bf003a0 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Mon, 23 Sep 2024 13:29:44 -0700 Subject: [PATCH 17/19] further rename update --- .../edge/support/create_bundle_int/test_secretstore_int.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_secretstore_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_secretstore_int.py index 8fca62a61..b3d880932 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/test_secretstore_int.py +++ b/azext_edge/tests/edge/support/create_bundle_int/test_secretstore_int.py @@ -14,7 +14,7 @@ def test_create_bundle_ssc(init_setup, tracked_files): """Test for ensuring file names and content. ONLY CHECKS arcagents.""" - ops_service = OpsServiceType.secretsynccontroller.value + ops_service = OpsServiceType.secretstore.value command = f"az iot ops support create-bundle --ops-service {ops_service}" walk_result, bundle_path = run_bundle_command(command=command, tracked_files=tracked_files) @@ -33,9 +33,9 @@ def test_create_bundle_ssc(init_setup, tracked_files): # SECRETSTORE expected_workload_types = ["deployment", "pod", "replicaset", "service"] expected_types = set(expected_workload_types) - assert set(file_map[OpsServiceType.secretsynccontroller.value].keys()).issubset(expected_types) + assert set(file_map[OpsServiceType.secretstore.value].keys()).issubset(expected_types) check_workload_resource_files( - file_objs=file_map[OpsServiceType.secretsynccontroller.value], + file_objs=file_map[OpsServiceType.secretstore.value], expected_workload_types=expected_workload_types, prefixes=["secrets-store-sync-controller-manager", "manager-metrics-service"], bundle_path=bundle_path, From 17009615d860e6cbfc6f60341c5ff22ec5a1d972 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Mon, 23 Sep 2024 16:27:14 -0700 Subject: [PATCH 18/19] update process_top_levels return type --- .../edge/support/create_bundle_int/helpers.py | 33 ++++++++----------- .../create_bundle_int/test_auto_int.py | 14 ++++---- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index 4392c2d9b..8fb4aee15 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -39,14 +39,6 @@ ] -class NamespaceTuple(NamedTuple): - arc: str - aio: str - acs: str - ssc: str - usage_system: str - - def assert_file_names(files: List[str]): """Asserts file names.""" for full_name in files: @@ -240,9 +232,12 @@ def get_file_map( mq_traces: bool = False, ) -> Dict[str, Dict[str, List[Dict[str, str]]]]: # Remove all files that will not be checked - arc_namespace, aio_namespace, acs_namespace, ssc_namespace, c_namespace = process_top_levels( - walk_result, ops_service - ) + namespaces = process_top_levels(walk_result, ops_service) + arc_namespace = namespaces.get("arc") + aio_namespace = namespaces.get("aio") + acs_namespace = namespaces.get("acs") + ssc_namespace = namespaces.get("ssc") + c_namespace = namespaces.get("usage_system") if aio_namespace: walk_result.pop(path.join(BASE_ZIP_PATH, aio_namespace)) @@ -308,7 +303,7 @@ def get_file_map( def process_top_levels( walk_result: Dict[str, Dict[str, List[str]]], ops_service: str, -) -> NamespaceTuple: +) -> Dict[str, Union[str, None]]: level_0 = walk_result.pop(BASE_ZIP_PATH) for file in ["events.yaml", "nodes.yaml", "storage-classes.yaml", "azure-clusterconfig.yaml"]: assert file in level_0["files"] @@ -374,13 +369,13 @@ def _get_namespace_determinating_files(name: str, folder: str, file_prefix: str) logger.debug(f"ACS namespace: {acs_namespace}") logger.debug(f"SSC namespace: {ssc_namespace}") - return NamespaceTuple( - arc=arc_namespace, - aio=namespace, - acs=acs_namespace, - ssc=ssc_namespace, - usage_system=clusterconfig_namespace, - ) + return { + "arc": arc_namespace, + "aio": namespace, + "acs": acs_namespace, + "ssc": ssc_namespace, + "usage_system": clusterconfig_namespace, + } def run_bundle_command( diff --git a/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py b/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py index 04cd9bdaf..7c304f3c4 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py +++ b/azext_edge/tests/edge/support/create_bundle_int/test_auto_int.py @@ -52,8 +52,10 @@ def test_create_bundle(init_setup, bundle_dir, mq_traces, ops_service, tracked_f ) # Level 0 - top - namespace = process_top_levels(walk_result, ops_service) - aio_namespace = namespace.aio + namespaces = process_top_levels(walk_result, ops_service) + aio_namespace = namespaces.get("aio") + acs_namespace = namespaces.get("acs") + ssc_namespace = namespaces.get("ssc") # Level 1 level_1 = walk_result.pop(path.join(BASE_ZIP_PATH, aio_namespace)) @@ -74,12 +76,12 @@ def test_create_bundle(init_setup, bundle_dir, mq_traces, ops_service, tracked_f walk_result[path.join(BASE_ZIP_PATH, aio_namespace, OpsServiceType.mq.value)]["folders"] = [] # remove acs resources from walk_result from aio namespace assertion - if namespace.acs: - walk_result.pop(path.join(BASE_ZIP_PATH, namespace.acs, "arccontainerstorage"), {}) + if acs_namespace: + walk_result.pop(path.join(BASE_ZIP_PATH, acs_namespace, "arccontainerstorage"), {}) # remove ssc resources in ssc namespace from walk_result from aio namespace assertion - if namespace.ssc: - walk_result.pop(path.join(BASE_ZIP_PATH, namespace.ssc, OpsServiceType.secretstore.value), {}) + if ssc_namespace: + walk_result.pop(path.join(BASE_ZIP_PATH, ssc_namespace, OpsServiceType.secretstore.value), {}) # Level 2 and 3 - bottom is_billing_included = OpsServiceType.billing.value in expected_services From e570616f557fe165ed608f5069072994187bd6a7 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Mon, 23 Sep 2024 16:50:32 -0700 Subject: [PATCH 19/19] fix lint --- azext_edge/tests/edge/support/create_bundle_int/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azext_edge/tests/edge/support/create_bundle_int/helpers.py b/azext_edge/tests/edge/support/create_bundle_int/helpers.py index 8fb4aee15..28081c02b 100644 --- a/azext_edge/tests/edge/support/create_bundle_int/helpers.py +++ b/azext_edge/tests/edge/support/create_bundle_int/helpers.py @@ -5,7 +5,7 @@ # ---------------------------------------------------------------------------------------------- from knack.log import get_logger -from typing import Dict, List, NamedTuple, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union from os import path from zipfile import ZipFile import pytest