Skip to content

Commit

Permalink
(feat): add workload identity in capz
Browse files Browse the repository at this point in the history
Signed-off-by: Ashutosh Kumar <[email protected]>
  • Loading branch information
sonasingh46 committed May 24, 2023
1 parent bee83d3 commit 22ccdf9
Show file tree
Hide file tree
Showing 17 changed files with 866 additions and 14 deletions.
12 changes: 10 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,11 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
SKIP_CLEANUP ?= false
SKIP_LOG_COLLECTION ?= false
SKIP_CREATE_MGMT_CLUSTER ?= false
# @sonasingh46: Skip creating mgmt cluster for ci as workload identity needs kind cluster
# to be created with extra mounts for key pairs which is not yet supported
# by existing e2e framework. A mgmt cluster(kind) is created as part of e2e suite
# that meets workload identity pre-requisites.
SKIP_CREATE_MGMT_CLUSTER ?= true
WIN_REPO_URL ?=

# Build time versioning details.
Expand Down Expand Up @@ -643,8 +647,12 @@ test-cover: test ## Run tests with code coverage and generate reports.
./hack/codecov-ignore.sh
go tool cover -html=coverage.out -o coverage.html

.PHONY: kind-create-bootstrap
kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
export AZWI=true KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh

.PHONY: test-e2e-run
test-e2e-run: generate-e2e-templates install-tools ## Run e2e tests.
test-e2e-run: generate-e2e-templates install-tools kind-create-bootstrap ## Run e2e tests.
$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
$(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
Expand Down
10 changes: 10 additions & 0 deletions Tiltfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ settings = {
"cert_manager_version": "v1.11.1",
"kubernetes_version": "v1.25.6",
"aks_kubernetes_version": "v1.25.6",
"azwi_version": "v1.1.0",
"flatcar_version": "3374.2.1",
}

Expand All @@ -46,6 +47,13 @@ if "allowed_contexts" in settings:
if "default_registry" in settings:
default_registry(settings.get("default_registry"))

# deploy AZWI webhook
def deploy_azwi():
version = settings.get("azwi_version")
azwi_uri = "https://github.com/Azure/azure-workload-identity/releases/download/{}/azure-wi-webhook.yaml".format(version)
cmd = "curl -sSL {} | {} | {} apply -f -".format(azwi_uri, envsubst_cmd, kubectl_cmd)
local(cmd, quiet = True)

# deploy CAPI
def deploy_capi():
version = settings.get("capi_version")
Expand Down Expand Up @@ -437,6 +445,8 @@ load("ext://cert_manager", "deploy_cert_manager")
if settings.get("deploy_cert_manager"):
deploy_cert_manager(version = settings.get("cert_manager_version"))

deploy_azwi()

deploy_capi()

create_identity_secret()
Expand Down
16 changes: 15 additions & 1 deletion api/v1beta1/azureclusteridentity_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ limitations under the License.
package v1beta1

import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/validation/field"
webhookutils "sigs.k8s.io/cluster-api-provider-azure/util/webhook"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)
Expand All @@ -40,7 +43,18 @@ func (c *AzureClusterIdentity) ValidateCreate() error {

// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
func (c *AzureClusterIdentity) ValidateUpdate(oldRaw runtime.Object) error {
return c.validateClusterIdentity()
var allErrs field.ErrorList
old := oldRaw.(*AzureClusterIdentity)
if err := webhookutils.ValidateImmutable(
field.NewPath("Spec", "Type"),
old.Spec.Type,
c.Spec.Type); err != nil {
allErrs = append(allErrs, err)
}
if len(allErrs) == 0 {
return c.validateClusterIdentity()
}
return apierrors.NewInvalid(GroupVersion.WithKind("AzureClusterIdentity").GroupKind(), c.Name, allErrs)
}

// ValidateDelete implements webhook.Validator so a webhook will be registered for the type.
Expand Down
5 changes: 4 additions & 1 deletion api/v1beta1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -535,7 +535,7 @@ const (
)

// IdentityType represents different types of identities.
// +kubebuilder:validation:Enum=ServicePrincipal;UserAssignedMSI;ManualServicePrincipal;ServicePrincipalCertificate
// +kubebuilder:validation:Enum=ServicePrincipal;UserAssignedMSI;ManualServicePrincipal;ServicePrincipalCertificate;WorkloadIdentity
type IdentityType string

const (
Expand All @@ -550,6 +550,9 @@ const (

// ServicePrincipalCertificate represents a service principal using a certificate as secret.
ServicePrincipalCertificate IdentityType = "ServicePrincipalCertificate"

// WorkloadIdentity represents a WorkloadIdentity.
WorkloadIdentity IdentityType = "WorkloadIdentity"
)

// OSDisk defines the operating system disk for a VM.
Expand Down
10 changes: 10 additions & 0 deletions azure/scope/identity.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,16 @@ func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceMa
var authErr error
var cred azcore.TokenCredential
switch p.Identity.Spec.Type {
case infrav1.WorkloadIdentity:
azwiCredOptions, err := NewWorkloadIdentityCredentialOptions().
WithTenantID(p.Identity.Spec.TenantID).
WithClientID(p.Identity.Spec.ClientID).
WithDefaults()
if err != nil {
return nil, errors.Wrapf(err, "failed to setup azwi options for identity %s", p.Identity.Name)
}
cred, authErr = NewWorkloadIdentityCredential(azwiCredOptions)

case infrav1.ServicePrincipal, infrav1.ServicePrincipalCertificate, infrav1.UserAssignedMSI:
if err := createAzureIdentityWithBindings(ctx, p.Identity, resourceManagerEndpoint, activeDirectoryEndpoint, clusterMeta, p.Client); err != nil {
return nil, err
Expand Down
155 changes: 155 additions & 0 deletions azure/scope/workload_identity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scope

import (
"context"
"os"
"strings"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/pkg/errors"
)

/*
Azure Workload Identity (AZWI) requires deploying AZWI mutating admission webhook
for self managed clusters e.g. Kind.
The webhook injects the following environment variables to the pod that
uses a label `azure.workload.identity/use=true`
|-----------------------------------------------------------------------------------|
|AZURE_AUTHORITY_HOST | The Azure Active Directory (AAD) endpoint. |
|AZURE_CLIENT_ID | The client ID of the Azure AD |
| | application or user-assigned managed identity. |
|AZURE_TENANT_ID | The tenant ID of the Azure subscription. |
|AZURE_FEDERATED_TOKEN_FILE | The path of the projected service account token file. |
|-----------------------------------------------------------------------------------|
In addition to the service account, it also projects a signed service account token to the
workload's volume(in this case the capz pod). The volume name is `azure-identity-token`
which is mounted at path `/var/run/secrets/azure/tokens/azure-identity-token` to the pod.
*/

const (
// AzureFedratedTokenFileEnvKey is the env key for AZURE_FEDERATED_TOKEN_FILE.
AzureFedratedTokenFileEnvKey = "AZURE_FEDERATED_TOKEN_FILE"
// AzureClientIDEnvKey is the env key for AZURE_CLIENT_ID.
AzureClientIDEnvKey = "AZURE_CLIENT_ID"
// AzureTenantIDEnvKey is the env key for AZURE_TENANT_ID.
AzureTenantIDEnvKey = "AZURE_TENANT_ID"
)

type workloadIdentityCredential struct {
assertion string
file string
cred *azidentity.ClientAssertionCredential
lastRead time.Time
}

// WorkloadIdentityCredentialOptions contains the configurable options for azwi.
type WorkloadIdentityCredentialOptions struct {
azcore.ClientOptions
ClientID string
TenantID string
TokenFilePath string
}

// NewWorkloadIdentityCredentialOptions returns an empty instance of WorkloadIdentityCredentialOptions.
func NewWorkloadIdentityCredentialOptions() *WorkloadIdentityCredentialOptions {
return &WorkloadIdentityCredentialOptions{}
}

// WithClientID sets client ID to WorkloadIdentityCredentialOptions.
func (w *WorkloadIdentityCredentialOptions) WithClientID(clientID string) *WorkloadIdentityCredentialOptions {
w.ClientID = clientID
return w
}

// WithTenantID sets tenant ID to WorkloadIdentityCredentialOptions.
func (w *WorkloadIdentityCredentialOptions) WithTenantID(tenantID string) *WorkloadIdentityCredentialOptions {
w.TenantID = tenantID
return w
}

// GetProjectedTokenPath return projected token file path from the env variable.
func GetProjectedTokenPath() (string, error) {
tokenPath := os.Getenv(AzureFedratedTokenFileEnvKey)
if strings.TrimSpace(tokenPath) == "" {
return "", errors.New("projected token path not injected")
}
return tokenPath, nil
}

// WithDefaults sets token file path. It also sets the client tenant ID from injected env in
// case empty values are passed.
func (w *WorkloadIdentityCredentialOptions) WithDefaults() (*WorkloadIdentityCredentialOptions, error) {
tokenFilePath, err := GetProjectedTokenPath()
if err != nil {
return nil, errors.Wrap(err, "failed to get token file path for identity")
}
w.TokenFilePath = tokenFilePath

// Fallback to using client ID from env variable if not set.
if strings.TrimSpace(w.ClientID) == "" {
w.ClientID = os.Getenv(AzureClientIDEnvKey)
if strings.TrimSpace(w.ClientID) == "" {
return nil, errors.New("empty client ID")
}
}

// // Fallback to using tenant ID from env variable.
if strings.TrimSpace(w.TenantID) == "" {
w.TenantID = os.Getenv(AzureTenantIDEnvKey)
if strings.TrimSpace(w.TenantID) == "" {
return nil, errors.New("empty tenant ID")
}
}
return w, nil
}

// NewWorkloadIdentityCredential returns a workload identity credential.
func NewWorkloadIdentityCredential(options *WorkloadIdentityCredentialOptions) (azcore.TokenCredential, error) {
w := &workloadIdentityCredential{file: options.TokenFilePath}
cred, err := azidentity.NewClientAssertionCredential(options.TenantID, options.ClientID, w.getAssertion, &azidentity.ClientAssertionCredentialOptions{ClientOptions: options.ClientOptions})
if err != nil {
return nil, err
}
w.cred = cred
return w, nil
}

// GetToken returns the token for azwi.
func (w *workloadIdentityCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) {
return w.cred.GetToken(ctx, opts)
}

func (w *workloadIdentityCredential) getAssertion(context.Context) (string, error) {
if now := time.Now(); w.lastRead.Add(5 * time.Minute).Before(now) {
content, err := os.ReadFile(w.file)
if err != nil {
return "", err
}
w.assertion = string(content)
w.lastRead = now
}
return w.assertion, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ spec:
- UserAssignedMSI
- ManualServicePrincipal
- ServicePrincipalCertificate
- WorkloadIdentity
type: string
required:
- clientID
Expand Down
3 changes: 3 additions & 0 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ metadata:
namespace: system
labels:
control-plane: capz-controller-manager
#ToDo: (@sonasingh46): Remove this label as part of aad pod identity deprecation
aadpodidbinding: capz-controller-aadpodidentity-selector

spec:
selector:
matchLabels:
Expand All @@ -16,6 +18,7 @@ spec:
labels:
control-plane: capz-controller-manager
aadpodidbinding: capz-controller-aadpodidentity-selector
azure.workload.identity/use: "true"
annotations:
kubectl.kubernetes.io/default-container: manager
spec:
Expand Down
3 changes: 2 additions & 1 deletion config/rbac/service_account.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
name: manager
namespace: system
namespace: system
Loading

0 comments on commit 22ccdf9

Please sign in to comment.