S3 user data support for AWSMachinePool (#592 et al)

api/v1beta2/tags.go

@@ -195,6 +195,12 @@ const (
 	// of the bootstrap secret that was used to create the user data for the latest launch
 	// template version.
 	LaunchTemplateBootstrapDataSecret = NameAWSProviderPrefix + "bootstrap-data-secret"
+
+	// LaunchTemplateBootstrapDataHash is the tag we use to store the hash of the raw bootstrap data.
+	// If bootstrap data is stored in S3, this hash relates to that data, not to the EC2 instance
+	// user data which only references the S3 object. We store this tag on launch template versions
+	// so that S3 bootstrap data objects can be deleted when they get outdated.
+	LaunchTemplateBootstrapDataHash = NameAWSProviderPrefix + "bootstrap-data-hash"
 )
 
 // ClusterTagKey generates the key for resources associated with a cluster.

cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go

@@ -193,6 +193,7 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument {
 				"arn:*:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*",
 			},
 			Action: iamv1.Actions{
+				"autoscaling:CancelInstanceRefresh",
 				"autoscaling:CreateAutoScalingGroup",
 				"autoscaling:UpdateAutoScalingGroup",
 				"autoscaling:CreateOrUpdateTags",
@@ -292,10 +293,12 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument {
 				"s3:CreateBucket",
 				"s3:DeleteBucket",
 				"s3:GetObject",
-				"s3:PutObject",
 				"s3:DeleteObject",
+				"s3:ListBucket",
 				"s3:PutBucketPolicy",
 				"s3:PutBucketTagging",
+				"s3:PutLifecycleConfiguration",
+				"s3:PutObject",
 			},
 		})
 	}

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml

@@ -255,6 +255,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_allow_assume_role.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml

@@ -255,6 +255,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml

@@ -255,6 +255,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml

@@ -255,6 +255,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
@@ -298,10 +299,12 @@ Resources:
           - s3:CreateBucket
           - s3:DeleteBucket
           - s3:GetObject
-          - s3:PutObject
           - s3:DeleteObject
+          - s3:ListBucket
           - s3:PutBucketPolicy
           - s3:PutBucketTagging
+          - s3:PutLifecycleConfiguration
+          - s3:PutObject
           Effect: Allow
           Resource:
           - arn:*:s3:::cluster-api-provider-aws-*

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml

@@ -249,6 +249,7 @@ Resources:
           Resource:
           - '*'
         - Action:
+          - autoscaling:CancelInstanceRefresh
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml

@@ -883,6 +883,23 @@ spec:
                   after it enters the InService state.
                   If no value is supplied by user a default value of 300 seconds is set
                 type: string
+              ignition:
+                description: Ignition defined options related to the bootstrapping
+                  systems where Ignition is used.
+                properties:
+                  version:
+                    default: "2.3"
+                    description: Version defines which version of Ignition will be
+                      used to generate bootstrap data.
+                    enum:
+                    - "2.3"
+                    - "3.0"
+                    - "3.1"
+                    - "3.2"
+                    - "3.3"
+                    - "3.4"
+                    type: string
+                type: object
               maxSize:
                 default: 1
                 description: MaxSize defines the maximum size of the group.

exp/api/v1beta1/conversion.go

@@ -52,6 +52,9 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 	if restored.Spec.AvailabilityZoneSubnetType != nil {
 		dst.Spec.AvailabilityZoneSubnetType = restored.Spec.AvailabilityZoneSubnetType
 	}
+	if restored.Spec.Ignition != nil {
+		dst.Spec.Ignition = restored.Spec.Ignition
+	}
 
 	if restored.Spec.AWSLaunchTemplate.PrivateDNSName != nil {
 		dst.Spec.AWSLaunchTemplate.PrivateDNSName = restored.Spec.AWSLaunchTemplate.PrivateDNSName

exp/api/v1beta2/awsmachinepool_types.go

@@ -101,6 +101,10 @@ type AWSMachinePoolSpec struct {
 	// SuspendProcesses defines a list of processes to suspend for the given ASG. This is constantly reconciled.
 	// If a process is removed from this list it will automatically be resumed.
 	SuspendProcesses *SuspendProcessesTypes `json:"suspendProcesses,omitempty"`
+
+	// Ignition defined options related to the bootstrapping systems where Ignition is used.
+	// +optional
+	Ignition *infrav1.Ignition `json:"ignition,omitempty"`
 }
 
 // SuspendProcessesTypes contains user friendly auto-completable values for suspended process names.

exp/api/v1beta2/awsmachinepool_webhook.go

@@ -27,7 +27,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
-	"sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
+	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/feature"
 )
 
 var log = ctrl.Log.WithName("awsmachinepool-resource")
@@ -62,12 +63,12 @@ func (r *AWSMachinePool) validateRootVolume() field.ErrorList {
 		return allErrs
 	}
 
-	if v1beta2.VolumeTypesProvisioned.Has(string(r.Spec.AWSLaunchTemplate.RootVolume.Type)) && r.Spec.AWSLaunchTemplate.RootVolume.IOPS == 0 {
+	if infrav1.VolumeTypesProvisioned.Has(string(r.Spec.AWSLaunchTemplate.RootVolume.Type)) && r.Spec.AWSLaunchTemplate.RootVolume.IOPS == 0 {
 		allErrs = append(allErrs, field.Required(field.NewPath("spec.awsLaunchTemplate.rootVolume.iops"), "iops required if type is 'io1' or 'io2'"))
 	}
 
 	if r.Spec.AWSLaunchTemplate.RootVolume.Throughput != nil {
-		if r.Spec.AWSLaunchTemplate.RootVolume.Type != v1beta2.VolumeTypeGP3 {
+		if r.Spec.AWSLaunchTemplate.RootVolume.Type != infrav1.VolumeTypeGP3 {
 			allErrs = append(allErrs, field.Required(field.NewPath("spec.awsLaunchTemplate.rootVolume.throughput"), "throughput is valid only for type 'gp3'"))
 		}
 		if *r.Spec.AWSLaunchTemplate.RootVolume.Throughput < 0 {
@@ -86,12 +87,12 @@ func (r *AWSMachinePool) validateNonRootVolumes() field.ErrorList {
 	var allErrs field.ErrorList
 
 	for _, volume := range r.Spec.AWSLaunchTemplate.NonRootVolumes {
-		if v1beta2.VolumeTypesProvisioned.Has(string(volume.Type)) && volume.IOPS == 0 {
+		if infrav1.VolumeTypesProvisioned.Has(string(volume.Type)) && volume.IOPS == 0 {
 			allErrs = append(allErrs, field.Required(field.NewPath("spec.template.spec.nonRootVolumes.iops"), "iops required if type is 'io1' or 'io2'"))
 		}
 
 		if volume.Throughput != nil {
-			if volume.Type != v1beta2.VolumeTypeGP3 {
+			if volume.Type != infrav1.VolumeTypeGP3 {
 				allErrs = append(allErrs, field.Required(field.NewPath("spec.template.spec.nonRootVolumes.throughput"), "throughput is valid only for type 'gp3'"))
 			}
 			if *volume.Throughput < 0 {
@@ -162,6 +163,22 @@ func (r *AWSMachinePool) validateRefreshPreferences() field.ErrorList {
 	return allErrs
 }
 
+func (r *AWSMachinePool) ignitionEnabled() bool {
+	return r.Spec.Ignition != nil
+}
+
+func (r *AWSMachinePool) validateIgnition() field.ErrorList {
+	var allErrs field.ErrorList
+
+	// Feature gate is not enabled but ignition is enabled then send a forbidden error.
+	if !feature.Gates.Enabled(feature.BootstrapFormatIgnition) && r.ignitionEnabled() {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "ignition"),
+			"can be set only if the BootstrapFormatIgnition feature gate is enabled"))
+	}
+
+	return allErrs
+}
+
 // ValidateCreate will do any extra validation when creating a AWSMachinePool.
 func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
 	log.Info("AWSMachinePool validate create", "machine-pool", klog.KObj(r))
@@ -176,6 +193,7 @@ func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
 	allErrs = append(allErrs, r.validateAdditionalSecurityGroups()...)
 	allErrs = append(allErrs, r.validateSpotInstances()...)
 	allErrs = append(allErrs, r.validateRefreshPreferences()...)
+	allErrs = append(allErrs, r.validateIgnition()...)
 
 	if len(allErrs) == 0 {
 		return nil, nil
@@ -226,4 +244,8 @@ func (r *AWSMachinePool) Default() {
 		log.Info("DefaultInstanceWarmup is zero, setting 300 seconds as default")
 		r.Spec.DefaultInstanceWarmup.Duration = 300 * time.Second
 	}
+
+	if r.ignitionEnabled() && r.Spec.Ignition.Version == "" {
+		r.Spec.Ignition.Version = infrav1.DefaultIgnitionVersion
+	}
 }