Encryption operator (#1665)

* Switch to external encryption key operator

* Switch to external encryption key operator

* Switch to external encryption key operator

* Switch to external encryption key operator

* Switch to external encryption key operator

* Switch to external encryption key operator

* Switch to external encryption key operator

* Switch to external encryption key operator

* avoid saving azurecluster if not needed

* avoid saving azurecluster if not needed

* Updated changelog

* Updated changelog

CHANGELOG.md

@@ -7,10 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Changes to EncryptionConfig in order to work with `encryption-provider-operator`.
+
 ### Fixed
 
 - Add pause annotation before deleting old machinepool and azuremachinepool CRs during migration to non-exp.
 - Update ownerReference UIDs during migration to non-exp.
+- Avoid updating `AzureCluster` at every reconciliation loop in the `subnet` resource.
+- Avoid saving `AzureCluster` status if there are no changes to avoid useless reconciliation loops.
 
 ## [5.20.0] - 2022-06-07
 

service/controller/azurecluster/handler/azureclusterconditions/create.go

@@ -2,6 +2,7 @@ package azureclusterconditions
 
 import (
 	"context"
+	"reflect"
 
 	"github.com/giantswarm/microerror"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -16,19 +17,24 @@ func (r *Resource) EnsureCreated(ctx context.Context, cr interface{}) error {
 		return microerror.Mask(err)
 	}
 
+	oldStatus := azureCluster.Status
+
 	// ensure Ready condition
 	err = r.ensureReadyCondition(ctx, &azureCluster)
 	if err != nil {
 		return microerror.Mask(err)
 	}
 
-	err = r.ctrlClient.Status().Update(ctx, &azureCluster)
-	if apierrors.IsConflict(err) {
-		r.logger.Debugf(ctx, "conflict trying to save object in k8s API concurrently")
-		r.logger.Debugf(ctx, "canceling resource")
-		return nil
-	} else if err != nil {
-		return microerror.Mask(err)
+	if !reflect.DeepEqual(oldStatus, azureCluster.Status) {
+		r.logger.Debugf(ctx, "status is changed, updating AzureCluster CR")
+		err = r.ctrlClient.Status().Update(ctx, &azureCluster)
+		if apierrors.IsConflict(err) {
+			r.logger.Debugf(ctx, "conflict trying to save object in k8s API concurrently")
+			r.logger.Debugf(ctx, "canceling resource")
+			return nil
+		} else if err != nil {
+			return microerror.Mask(err)
+		}
 	}
 
 	return nil

service/controller/azurecluster/handler/subnet/resource.go

@@ -207,7 +207,19 @@ func (r *Resource) ensureSubnets(ctx context.Context, deploymentsClient *azurere
 				return microerror.Mask(err)
 			}
 
-			azureCluster.Spec.NetworkSpec.Subnets[i].ID = subnetID
+			if azureCluster.Spec.NetworkSpec.Subnets[i].ID != subnetID {
+				azureCluster.Spec.NetworkSpec.Subnets[i].ID = subnetID
+
+				// Update AzureCluster so that subnet.ID is saved.
+				err = r.ctrlClient.Update(ctx, azureCluster)
+				if apierrors.IsConflict(err) {
+					r.logger.Debugf(ctx, "conflict trying to save object in k8s API concurrently")
+					r.logger.Debugf(ctx, "cancelling resource")
+					return nil
+				} else if err != nil {
+					return microerror.Mask(err)
+				}
+			}
 
 			err = r.ensureSubnetIsAllowedToStorageAccount(ctx, storageAccountsClient, azureCluster, azureCluster.Spec.NetworkSpec.Subnets[i])
 			if err != nil {
@@ -216,16 +228,6 @@ func (r *Resource) ensureSubnets(ctx context.Context, deploymentsClient *azurere
 		}
 	}
 
-	// Update AzureCluster so that subnet.ID is saved.
-	err = r.ctrlClient.Update(ctx, azureCluster)
-	if apierrors.IsConflict(err) {
-		r.logger.Debugf(ctx, "conflict trying to save object in k8s API concurrently")
-		r.logger.Debugf(ctx, "cancelling resource")
-		return nil
-	} else if err != nil {
-		return microerror.Mask(err)
-	}
-
 	return nil
 }
 

service/controller/azureconfig/handler/blobobject/desired.go

@@ -9,6 +9,7 @@ import (
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/operatorkit/v7/pkg/controller/context/resourcecanceledcontext"
 	"golang.org/x/sync/errgroup"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	capi "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -118,6 +119,21 @@ func (r *Resource) GetDesiredState(ctx context.Context, obj interface{}) (interf
 		return nil, microerror.Mask(err)
 	}
 
+	var encryptionConfig []byte
+	{
+		var secret corev1.Secret
+		err := r.ctrlClient.Get(
+			ctx, client.ObjectKey{
+				Name:      key.EncryptionConfigSecretName(key.ClusterID(&cr)),
+				Namespace: key.OrganizationNamespace(&cr),
+			},
+			&secret)
+		if err != nil {
+			return "", microerror.Mask(err)
+		}
+		encryptionConfig = secret.Data[encryptionProviderConfigKeyName]
+	}
+
 	var cluster capi.Cluster
 	{
 		cluster = capi.Cluster{}
@@ -149,6 +165,7 @@ func (r *Resource) GetDesiredState(ctx context.Context, obj interface{}) (interf
 			CustomObject:    cr,
 			Images:          images,
 			MasterCertFiles: masterCertFiles,
+			EncryptionConf:  encryptionConfig,
 			Versions:        versions,
 			WorkerCertFiles: workerCertFiles,
 		}

service/controller/azureconfig/handler/blobobject/resource.go

@@ -19,6 +19,8 @@ import (
 const (
 	// Name is the identifier of the resource.
 	Name = "blobobject"
+
+	encryptionProviderConfigKeyName = "encryption"
 )
 
 type Config struct {

service/controller/cloudconfig/cloud_config.go

@@ -1,18 +1,12 @@
 package cloudconfig
 
 import (
-	"context"
-
 	"github.com/Azure/go-autorest/autorest/azure/auth"
-	providerv1alpha1 "github.com/giantswarm/apiextensions/v6/pkg/apis/provider/v1alpha1"
-	apiextensionslabels "github.com/giantswarm/apiextensions/v6/pkg/label"
 	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v13/pkg/template"
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/micrologger"
-	corev1 "k8s.io/api/core/v1"
 	ctrl "sigs.k8s.io/controller-runtime/pkg/client"
 
-	"github.com/giantswarm/azure-operator/v5/service/controller/key"
 	"github.com/giantswarm/azure-operator/v5/service/controller/setting"
 )
 
@@ -23,11 +17,6 @@ const (
 	FileOwnerGroupName          = "root"
 	FileOwnerGroupIDNobody      = 65534
 	FilePermission              = 0700
-	// randomKeyLabel is the label used in the secret to identify a secret
-	// containing the random key.
-	randomKeyLabel      = "giantswarm.io/randomkey"
-	randomKeyLabelValue = "encryption"
-	secretKey           = "encryption"
 )
 
 type Key string
@@ -101,37 +90,6 @@ func New(config Config) (*CloudConfig, error) {
 	return c, nil
 }
 
-func (c CloudConfig) getEncryptionkey(ctx context.Context, customObject providerv1alpha1.AzureConfig) (string, error) {
-	secretList := &corev1.SecretList{}
-	{
-		c.logger.Debugf(ctx, "try to find encryption secret")
-		err := c.ctrlClient.List(
-			ctx,
-			secretList,
-			ctrl.MatchingLabels{
-				randomKeyLabel:              randomKeyLabelValue,
-				apiextensionslabels.Cluster: key.ClusterID(&customObject),
-			},
-		)
-		if err != nil {
-			return "", microerror.Mask(err)
-		}
-	}
-
-	if len(secretList.Items) > 0 {
-		c.logger.Debugf(ctx, "found encryption secret in namespace '%s/%s'", secretList.Items[0].Namespace, secretList.Items[0].Name)
-		randomkey, ok := secretList.Items[0].Data[secretKey]
-		if !ok {
-			return "", microerror.Maskf(invalidSecretError, "%q key missing", secretKey)
-		}
-
-		return string(randomkey), nil
-	}
-
-	return "", microerror.Mask(secretNotFoundError)
-
-}
-
 func newCloudConfig(template string, params k8scloudconfig.Params) (string, error) {
 	c := k8scloudconfig.CloudConfigConfig{
 		Params:   params,

service/controller/cloudconfig/master_template.go

@@ -18,16 +18,14 @@ const (
 	defaultEtcdPort                  = 2379
 	defaultImagePullProgressDeadline = "1m"
 	EtcdInitialClusterStateNew       = "new"
+
+	encryptionConfigFilePath = "/etc/kubernetes/encryption/k8s-encryption-config.yaml"
 )
 
 // NewMasterCloudConfig generates a new master cloudconfig and returns it as a
 // base64 encoded string.
 func (c CloudConfig) NewMasterTemplate(ctx context.Context, data IgnitionTemplateData, encrypter encrypter.Interface) (string, error) {
-	apiserverEncryptionKey, err := c.getEncryptionkey(ctx, data.CustomObject)
-	if err != nil {
-		return "", microerror.Mask(err)
-	}
-
+	var err error
 	var k8sAPIExtraArgs []string
 	{
 		oidcExtraArgs := c.oidcExtraArgs(ctx, data)
@@ -50,7 +48,6 @@ func (c CloudConfig) NewMasterTemplate(ctx context.Context, data IgnitionTemplat
 
 		params = k8scloudconfig.Params{}
 		params.BaseDomain = key.ClusterBaseDomain(data.CustomObject)
-		params.APIServerEncryptionKey = apiserverEncryptionKey
 		params.Cluster = data.CustomObject.Spec.Cluster
 		params.CalicoPolicyOnly = true
 		params.DockerhubToken = c.dockerhubToken
@@ -99,8 +96,14 @@ func (c CloudConfig) NewMasterTemplate(ctx context.Context, data IgnitionTemplat
 			},
 		}
 
+		encryptedEncryptionConfig, err := encrypter.Encrypt(data.EncryptionConf)
+		if err != nil {
+			return "", microerror.Mask(err)
+		}
+
 		params.Extension = &masterExtension{
-			baseExtension: be,
+			baseExtension:             be,
+			encryptedEncryptionConfig: encryptedEncryptionConfig,
 		}
 		params.ExtraManifests = []string{}
 		params.Debug = k8scloudconfig.Debug{
@@ -112,6 +115,7 @@ func (c CloudConfig) NewMasterTemplate(ctx context.Context, data IgnitionTemplat
 		params.RegistryMirrors = c.registryMirrors
 		params.Versions = data.Versions
 		params.SSOPublicKey = c.ssoPublicKey
+		params.DisableEncryptionAtREST = true // encryption key is now managed by encription-provider-operator
 	}
 	ignitionPath := k8scloudconfig.GetIgnitionPath(c.ignition.Path)
 	params.Files, err = k8scloudconfig.RenderFiles(ignitionPath, params)
@@ -124,6 +128,8 @@ func (c CloudConfig) NewMasterTemplate(ctx context.Context, data IgnitionTemplat
 
 type masterExtension struct {
 	baseExtension
+
+	encryptedEncryptionConfig []byte
 }
 
 // oidcExtraArgs returns oidc parameters reading the configuration from `Cluster` annotations.
@@ -265,6 +271,28 @@ func (me *masterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 		fileAssets = append(fileAssets, asset)
 	}
 
+	// Add encryption at rest config.
+	{
+		m := k8scloudconfig.FileMetadata{
+			Path: encryptionConfigFilePath + ".enc",
+			Owner: k8scloudconfig.Owner{
+				Group: k8scloudconfig.Group{
+					Name: FileOwnerGroupName,
+				},
+				User: k8scloudconfig.User{
+					Name: FileOwnerUserName,
+				},
+			},
+			Permissions: CertFilePermission,
+		}
+
+		asset := k8scloudconfig.FileAsset{
+			Metadata: m,
+			Content:  base64.StdEncoding.EncodeToString(me.encryptedEncryptionConfig),
+		}
+
+		fileAssets = append(fileAssets, asset)
+	}
 	return fileAssets, nil
 }
 
@@ -305,6 +333,9 @@ func (me *masterExtension) Units() ([]k8scloudconfig.UnitAsset, error) {
 
 	data := me.templateData(me.certFiles)
 
+	// To use the certificate decrypter unit for the etcd data encryption config file.
+	data.certificateDecrypterUnitParams.CertsPaths = append(data.certificateDecrypterUnitParams.CertsPaths, encryptionConfigFilePath)
+
 	var newUnits []k8scloudconfig.UnitAsset
 
 	for _, fm := range unitsMeta {

service/controller/cloudconfig/types.go

@@ -56,6 +56,7 @@ type IgnitionTemplateData struct {
 	AzureCluster     *capz.AzureCluster
 	Cluster          *capi.Cluster
 	CustomObject     providerv1alpha1.AzureConfig
+	EncryptionConf   []byte
 	Images           k8scloudconfig.Images
 	MachinePool      *capiexp.MachinePool
 	MasterCertFiles  []certs.File

service/controller/key/common.go

@@ -92,3 +92,7 @@ func KubernetesVersion(release releasev1alpha1.Release) (string, error) {
 func OSVersion(release releasev1alpha1.Release) (string, error) {
 	return ComponentVersion(release, ComponentOS)
 }
+
+func EncryptionConfigSecretName(clusterName string) string {
+	return fmt.Sprintf("%s-encryption-provider-config", clusterName)
+}