test: add integration testing for az iot ops secretsync

.github/actions/connect-arc/action.yml

@@ -5,7 +5,7 @@
 
 name: 'Connect your cluster to Azure ARC'
 description: |
-  Action to connect your kubernetes cluster to Azure ARC using `az connectedk8s connect`. 
+  Action to connect your kubernetes cluster to Azure ARC using `az connectedk8s connect`.
   Will add the connectedk8s Azure CLI extension if not already installed.
 inputs:
   cluster-name:
@@ -18,10 +18,12 @@ inputs:
     description: Contents of kubeconfig to connect to cluster. If not provided, `~/.kube/config` will be used.
   custom-locations-oid:
     description: |
-      Object ID of Custom Locations Application in your tenant. 
+      Object ID of Custom Locations Application in your tenant.
       Find with: `az ad sp show --id bc313c14-388c-4e7d-a58e-70017303ee3b --query id -o tsv`.
-      
+
       Please provide this value if your azure login context lacks permission to make this call.
+  enable-workload-identity:
+    description: Enable workload identity so secret sync can be applied.
 
 runs:
   using: 'composite'
@@ -52,13 +54,24 @@ runs:
         -g ${{ env.resource-group }}
         ${{ env.custom-locations-oid }}
         ${{ env.ARC_CLUSTER_CONFIG && format('--kube-config {0}', env.ARC_CLUSTER_CONFIG) || '' }}
-        
+
         az connectedk8s enable-features --features custom-locations cluster-connect
         -n ${{ env.cluster-name }}
         -g ${{ env.resource-group }}
         ${{ env.custom-locations-oid }}
         ${{ env.ARC_CLUSTER_CONFIG && format('--kube-config {0}', env.ARC_CLUSTER_CONFIG) || '' }}
       shell: bash
+    - name: "Update cluster to enable workload identity"
+      if: inputs.enable-workload-identity
+      env:
+        cluster-name: "${{ inputs.cluster-name }}"
+        resource-group: "${{ inputs.resource-group }}"
+      run: >-
+        az connectedk8s update
+        -n ${{ env.cluster-name }}
+        -g ${{ env.resource-group }}
+        --enable-oidc-issuer --enable-workload-identity
+      shell: bash
     - name: "Wait for cluster connected status"
       env:
         cluster-name: "${{ inputs.cluster-name }}"
@@ -67,9 +80,9 @@ runs:
         retry_count=0
         max_retries=5
         sleep=5
-        
+
         status=$(az connectedk8s show -n ${{ env.cluster-name }} -g ${{ env.resource-group }} --query "connectivityStatus" -o tsv)
-        
+
         while [ "$status" != "Connected" ] && [ $retry_count -lt $max_retries ]; do
           echo "Cluster status is "$status", Retrying in $sleep seconds..."
           sleep $sleep

.github/workflows/int_test.yml

@@ -88,6 +88,7 @@ jobs:
     env:
       CLUSTER_NAME: "opt${{ github.run_number }}${{ matrix.feature }}"
       INSTANCE_NAME: "inst${{ github.run_number }}${{ matrix.feature }}"
+      KEYVAULT_NAME: "kv${{ github.run_number }}"
     strategy:
       fail-fast: false
       matrix:
@@ -224,6 +225,18 @@ jobs:
           spec:
             selfSigned: {}
           EOF
+      - name: "Tox test environment setup for init"
+        run: |
+          python -m pip install tox
+          tox r -vv -e python-init-int --notest
+      - name: "Tox test environment setup for integration tests"
+        if: ${{ matrix.feature == 'default' && !inputs.use-container }}
+        run: |
+          tox r -vv -e "python-all-int" --notest
+      - name: "Tox test environment setup for integration tests"
+        if: ${{ matrix.feature == 'secretsync' && !inputs.use-container }}
+        run: |
+          tox r -vv -e "python-secretsync-int" --notest
       - name: "Az CLI login"
         uses: azure/login@v2
         with:
@@ -236,14 +249,13 @@ jobs:
           cluster-name: ${{ env.CLUSTER_NAME }}
           resource-group: ${{ env.RESOURCE_GROUP }}
           custom-locations-oid: ${{ env.CUSTOM_LOCATIONS_OID }}
-      - name: "Tox test environment setup for init"
-        run: |
-          python -m pip install tox
-          tox r -vv -e python-init-int --notest
-      - name: "Tox test environment setup for integration tests"
-        if: ${{ matrix.feature == 'default' && !inputs.use-container }}
-        run: |
-          tox r -vv -e "python-all-int" --notest
+          enable-workload-identity: ${{ matrix.feature == 'trustbundle' }}
+      - name: "Az CLI login"
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
       - name: "Tox INIT Integration Tests"
         env:
           AIO_CLI_INIT_PREFLIGHT_DISABLED: ${{ steps.init.outputs.NO_PREFLIGHT }}
@@ -267,6 +279,16 @@ jobs:
           tox r -e "python-all-int" --skip-pkg-install -- --durations=0 --dist=loadfile -n auto
           coverage=$(jq .totals.percent_covered coverage.json | cut -c1-4)
           echo "Code coverage: $coverage%" >> $GITHUB_STEP_SUMMARY
+      - name: "Tox Workload Identity Integration Tests"
+        if: ${{ matrix.feature == 'trustbundle' && !inputs.use-container }}
+        env:
+          azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }}
+          azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }}
+          azext_edge_sp_object_id: ${{ secrets.AZURE_CLIENT_ID }}
+        run: |
+          tox r -e "python-wlif-int" --skip-pkg-install -- --durations=0 --dist=loadfile -n auto
+          coverage=$(jq .totals.percent_covered coverage.json | cut -c1-4)
+          echo "Code coverage: $coverage%" >> $GITHUB_STEP_SUMMARY
       - name: "Containerized tests"
         if: ${{ matrix.feature == 'default' && inputs.use-container }}
         env:

azext_edge/tests/edge/orchestration/test_secretsync_int.py

@@ -0,0 +1,246 @@
+# coding=utf-8
+# ----------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License file in the project root for license information.
+# ----------------------------------------------------------------------------------------------
+
+import pytest
+from knack.log import get_logger
+from time import sleep
+from typing import Optional
+
+from azure.cli.core.azclierror import CLIInternalError
+from ...generators import generate_random_string
+from ...helpers import run
+
+logger = get_logger(__name__)
+ROLE_MAX_RETRIES = 5
+ROLE_RETRY_INTERVAL = 15
+
+
+@pytest.fixture
+def secretsync_int_setup(settings, tracked_resources):
+    from ...settings import EnvironmentVariables
+
+    settings.add_to_config(EnvironmentVariables.rg.value)
+    settings.add_to_config(EnvironmentVariables.instance.value)
+    settings.add_to_config(EnvironmentVariables.kv.value)
+    settings.add_to_config(EnvironmentVariables.user_assigned_mi_id.value)
+    settings.add_to_config(EnvironmentVariables.sp_object_id.value)
+
+    if not all([settings.env.azext_edge_instance, settings.env.azext_edge_rg]):
+        raise AssertionError(
+            f"Cannot run secretsync tests without an instance and resource group. Current settings:\n {settings}"
+        )
+    if not any([settings.env.azext_edge_kv, settings.env.azext_edge_sp_object_id]):
+        pytest.skip(
+            "Cannot run secretsync tests without a keyvault id or a object id. Object Id is needed to add "
+            "'Key Vault Secrets Officer' to a newly created key vault."
+        )
+
+    kv_id = settings.env.azext_edge_kv
+    kv_name = None
+    if not kv_id:
+        kv_name = "spc" + generate_random_string(size=6)
+        kv_id = run(f"az keyvault create -n {kv_name} -g {settings.env.azext_edge_rg}")["id"]
+        # add "Key Vault Secrets Officer" role
+        run(
+            "az role assignment create --role b86a8fe4-44ce-4948-aee5-eccb2c155cd7 "
+            f"--assignee {settings.env.azext_edge_sp_object_id} --scope {kv_id}"
+        )
+
+    mi_id = settings.env.azext_edge_user_assigned_mi_id
+    if not mi_id:
+        mi_id = run(
+            f"az identity create -n {'spc' + generate_random_string(size=6)} -g {settings.env.azext_edge_rg}"
+        )["id"]
+        tracked_resources.append(mi_id)
+
+    instance_name = settings.env.azext_edge_instance
+    resource_group = settings.env.azext_edge_rg
+    # list to track initial result if there is something
+    initial_list_result = run(f"az iot ops secretsync list -n {instance_name} -g {resource_group}")
+    if initial_list_result:
+        run(f"az iot ops secretsync disable -n {instance_name} -g {resource_group} -y")
+
+    yield {
+        "resourceGroup": resource_group,
+        "instanceName": instance_name,
+        "keyvaultId": kv_id,
+        "userAssignedId": mi_id,
+    }
+
+    # note that you need to purge the kv too...
+    if kv_name:
+        try:
+            run(f"az keyvault delete -n {kv_name} -g {settings.env.azext_edge_rg}")
+            # sometimes it takes a bit to get the deleted list to update
+            sleep(ROLE_RETRY_INTERVAL)
+            run(f"az keyvault purge -n {kv_name}")
+        except CLIInternalError as e:
+            logger.error(f"Failed to delete the keyvault {kv_name} properly. {e.error_msg}")
+
+    # if it was enabled before, reenable
+    if initial_list_result:
+        kv_name = initial_list_result[0]["properties"]["keyvaultName"]
+        mi_client_id = initial_list_result[0]["properties"]["clientId"]
+        spc_name = initial_list_result[0]["name"]
+        try:
+            kv_id = run(f"az keyvault show -n {kv_name}")["id"]
+            mi_id = run(f"az identity list --query \"[?clientId=='{mi_client_id}']\"")[0]["id"]
+            # if the role assignments were applied, they should still exist
+            # TODO: phase 2 - direct cluster connection for --self-hosted-issuer
+            run(
+                f"az iot ops secretsync enable -n {instance_name} -g {resource_group} "
+                f"--mi-user-assigned {mi_id} --kv-resource-id {kv_id} --spc {spc_name} --skip-ra"
+            )
+        except (CLIInternalError, IndexError):
+            logger.error("Could not reenable secretsync correctly.")
+
+
+@pytest.mark.rpsaas
+@pytest.mark.require_wlif_setup
+def test_secretsync(secretsync_int_setup):
+    resource_group = secretsync_int_setup["resourceGroup"]
+    instance_name = secretsync_int_setup["instanceName"]
+    kv_id = secretsync_int_setup["keyvaultId"]
+    mi_id = secretsync_int_setup["userAssignedId"]
+    use_self_hosted_issuer = False
+
+    extended_loc = run(f"az iot ops show -g {resource_group} -n {instance_name}")["extendedLocation"]["name"]
+    mi_client_id = run(f"az identity show --ids {mi_id}")["clientId"]
+    mi_principal_id = run(f"az identity show --ids {mi_id}")["principalId"]
+    expected_result = {
+        "extended_location": extended_loc,
+        "resource_group": resource_group,
+        "kv_name": kv_id.rsplit("/", maxsplit=1)[-1],
+        "mi_client_id": mi_client_id,
+    }
+
+    initial_role_list = [
+        role["roleDefinitionName"] for role in _get_role_list(kv_id, mi_client_id)
+    ]
+
+    # enable with skip ra + check if test can be run
+    try:
+        enable_result = run(
+            f"az iot ops secretsync enable -n {instance_name} -g {resource_group} "
+            f"--mi-user-assigned {mi_id} --kv-resource-id {kv_id} --skip-ra"
+        )
+    except CLIInternalError as e:
+        if "not enabled as an oidc issuer or for workload identity federation." in e.error_msg:
+            pytest.skip("Cluster is not enabled for secretsync.")
+        elif "No issuerUrl is available." in e.error_msg:
+            use_self_hosted_issuer = True
+            enable_result = run(
+                f"az iot ops secretsync enable -n {instance_name} -g {resource_group} "
+                f"--mi-user-assigned {mi_id} --kv-resource-id {kv_id} --skip-ra --self-hosted-issuer"
+            )
+        else:
+            raise e
+    _assert_secret_sync_class(
+        result=enable_result,
+        **expected_result
+    )
+    _assert_role_assignments(
+        initial_assignment_names=initial_role_list,
+        kv_id=kv_id,
+        mi_principal_id=mi_principal_id
+    )
+
+    # list
+    list_result = run(f"az iot ops secretsync list -n {instance_name} -g {resource_group}")
+    assert len(list_result) == 1
+    _assert_secret_sync_class(
+        result=list_result[0],
+        **expected_result
+    )
+
+    # disable
+    run(f"az iot ops secretsync disable -n {instance_name} -g {resource_group} -y")
+
+    # second enable with custom name
+    spc_name = generate_random_string(force_lower=True)
+    enable_result = run(
+        f"az iot ops secretsync enable -n {instance_name} -g {resource_group} "
+        f"--mi-user-assigned {mi_id} --kv-resource-id {kv_id} --spc {spc_name} "
+        f"--skip-ra false {'--self-hosted-issuer' if use_self_hosted_issuer else ''} "
+    )
+    # TODO: phase 2 - direct cluster connection for --self-hosted-issuer
+    _assert_secret_sync_class(
+        result=enable_result,
+        spc_name=spc_name,
+        **expected_result
+    )
+    _assert_role_assignments(
+        initial_assignment_names=initial_role_list,
+        kv_id=kv_id,
+        mi_principal_id=mi_principal_id,
+        expected_secretsync_roles=True
+    )
+
+    # disable
+    run(f"az iot ops secretsync disable -n {instance_name} -g {resource_group} -y")
+
+
+def _assert_secret_sync_class(
+    result: dict,
+    extended_location: str,
+    resource_group: str,
+    kv_name: str,
+    mi_client_id: str,
+    spc_name: Optional[str] = None,
+):
+    assert result["extendedLocation"]["name"] == extended_location
+    assert result["resourceGroup"] == resource_group
+    if spc_name:
+        assert result["name"] == spc_name
+    else:
+        assert result["name"].startswith("spc-ops-")
+
+    assert result["properties"]["keyvaultName"] == kv_name
+    assert result["properties"]["clientId"] == mi_client_id
+
+
+def _assert_role_assignments(
+    initial_assignment_names: list,
+    kv_id: str,
+    mi_principal_id: str,
+    expected_secretsync_roles: bool = False
+):
+    tries = 0
+    while tries < ROLE_MAX_RETRIES:
+        try:
+            current_assignment_names = [
+                role["roleDefinitionName"] for role in run(
+                    f"az role assignment list --scope {kv_id} --assignee {mi_principal_id}"
+                )
+            ]
+            if expected_secretsync_roles:
+                assert "Key Vault Secrets User" in current_assignment_names
+                assert "Key Vault Reader" in current_assignment_names
+            else:
+                # role could have been applied before - so just make sure nothing new was applied
+                difference_roles = set(current_assignment_names).difference(set(initial_assignment_names))
+                assert not difference_roles
+            return
+        except AssertionError as e:
+            tries += 1
+            sleep(ROLE_RETRY_INTERVAL)
+            if tries == ROLE_MAX_RETRIES:
+                raise e
+
+
+def _get_role_list(
+    kv_id: str,
+    mi_client_id: str
+):
+    tries = 0
+    while tries < ROLE_MAX_RETRIES:
+        try:
+            return run(f"az role assignment list --scope {kv_id} --assignee {mi_client_id}")
+        except CLIInternalError:
+            tries += 1
+            sleep(ROLE_RETRY_INTERVAL)
+
+    raise AssertionError("Failed to create user assigned identity. Please retry with a given identity.")

azext_edge/tests/helpers.py

@@ -165,6 +165,7 @@ def run(command: str, shell_mode: bool = True, expect_failure: bool = False):
     Wrapper function for run_host_command used for testing.
     Parameter `expect_failure` determines if an error will be raised for the command result.
     The output is converted to non-binary text and loaded as a json if possible.
+    Raises CLIInternalError if there is an unexpected error or an unexpected success.
     """
     import subprocess
 

azext_edge/tests/settings.py

@@ -16,6 +16,7 @@ class EnvironmentVariables(Enum):
     instance = "azext_edge_instance"
     context_name = "azext_edge_context_name"
     kv = "azext_edge_kv"
+    user_assigned_mi_id = "azext_edge_user_assigned_mi_id"
     sp_app_id = "azext_edge_sp_app_id"
     sp_object_id = "azext_edge_sp_object_id"
     sp_secret = "azext_edge_sp_secret"

pytest.ini.example

@@ -23,3 +23,4 @@ markers =
     rpsaas: mark tests that are purely cloud-side
     e2e: mark tests that run in e2e pipeline
     upgrade_scenario_test: mark tests that will run az iot ops upgrade
+    require_wlif_setup: mark tests that require a workload identity setup for the cluster