feat(eks): spot support for managed nodegroups (#11962)

This PR adds the `CapacityType` support and allows users to create Spot managed node groups for Amazon EKS.

1. The `CapacityType` attribute is supported by cloudformation but not yet documented. We tentatively use addPropertyOverride() to enable it.
2. `instanceType` will be deprecated and we introduced the new `instanceTypes`
3. `instanceTypes` with different CPU architectures will throw an error.
4. `amiType` is still optional, however, when specified, incorrect `amiType` will throw the error.
5. According to the [document](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-instancetypes), we are allowed to specify instance type(s) in either `instanceTypes` property or launch template but not both. As we can't check the content of the launch template passed in, we allow `instanceTypes` and launch template both specified and encourage to use `instanceTypes` when possible.


## Sample

```ts
cluster.addNodegroupCapacity('extra-ng-spot', {
  instanceTypes: [
    new ec2.InstanceType('c5.large'),
    new ec2.InstanceType('c5a.large'),
    new ec2.InstanceType('c5d.large'),
  ],
  minSize: 3,
  capacityType: eks.CapacityType.SPOT,
});
```

Closes #11827 

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: pahud

packages/@aws-cdk/aws-eks/README.md

@@ -201,14 +201,35 @@ const cluster = new eks.Cluster(this, 'HelloEKS', {
 });
 
 cluster.addNodegroupCapacity('custom-node-group', {
-  instanceType: new ec2.InstanceType('m5.large'),
+  instanceTypes: [new ec2.InstanceType('m5.large')],
   minSize: 4,
   diskSize: 100,
   amiType: eks.NodegroupAmiType.AL2_X86_64_GPU,
   ...
 });
 ```
 
+#### Spot Instances Support
+
+Use `capacityType` to create managed node groups comprised of spot instances. To maximize the availability of your applications while using
+Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property. 
+
+> For more details visit [Managed node group capacity types](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types).
+
+
+```ts
+cluster.addNodegroupCapacity('extra-ng-spot', {
+  instanceTypes: [
+    new ec2.InstanceType('c5.large'),
+    new ec2.InstanceType('c5a.large'),
+    new ec2.InstanceType('c5d.large'),
+  ],
+  minSize: 3,
+  capacityType: eks.CapacityType.SPOT,
+});
+
+```
+
 #### Launch Template Support
 
 You can specify a launch template that the node group will use. Note that when using a custom AMI, Amazon EKS doesn't merge any user data.
@@ -236,7 +257,9 @@ cluster.addNodegroupCapacity('extra-ng', {
 });
 ```
 
-> For more details visit [Launch Template Support](https://docs.aws.amazon.com/en_ca/eks/latest/userguide/launch-templates.html).
+You may specify one or instance types in either the `instanceTypes` property of `NodeGroup` or in the launch template, **but not both**.
+
+> For more details visit [Launch Template Support](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html).
 
 Graviton 2 instance types are supported including `c6g`, `m6g`, `r6g` and `t4g`.
 
@@ -552,7 +575,7 @@ Amazon Linux 2 AMI for ARM64 will be automatically selected.
 ```ts
 // add a managed ARM64 nodegroup
 cluster.addNodegroupCapacity('extra-ng-arm', {
-  instanceType: new ec2.InstanceType('m6g.medium'),
+  instanceTypes: [new ec2.InstanceType('m6g.medium')],
   minSize: 2,
 });
 

packages/@aws-cdk/aws-eks/lib/cluster.ts

@@ -1123,7 +1123,7 @@ export class Cluster extends ClusterBase {
         this.addAutoScalingGroupCapacity('DefaultCapacity', { instanceType, minCapacity }) : undefined;
 
       this.defaultNodegroup = props.defaultCapacityType !== DefaultCapacityType.EC2 ?
-        this.addNodegroupCapacity('DefaultCapacity', { instanceType, minSize: minCapacity }) : undefined;
+        this.addNodegroupCapacity('DefaultCapacity', { instanceTypes: [instanceType], minSize: minCapacity }) : undefined;
     }
 
     const outputConfigCommand = props.outputConfigCommand === undefined ? true : props.outputConfigCommand;

packages/@aws-cdk/aws-eks/lib/managed-nodegroup.ts

@@ -1,6 +1,6 @@
 import { InstanceType, ISecurityGroup, SubnetSelection } from '@aws-cdk/aws-ec2';
 import { IRole, ManagedPolicy, Role, ServicePrincipal } from '@aws-cdk/aws-iam';
-import { IResource, Resource } from '@aws-cdk/core';
+import { IResource, Resource, Annotations } from '@aws-cdk/core';
 import { Construct } from 'constructs';
 import { Cluster, ICluster } from './cluster';
 import { CfnNodegroup } from './eks.generated';
@@ -37,6 +37,20 @@ export enum NodegroupAmiType {
   AL2_ARM_64 = 'AL2_ARM_64'
 }
 
+/**
+ * Capacity type of the managed node group
+ */
+export enum CapacityType {
+  /**
+   * spot instances
+   */
+  SPOT = 'SPOT',
+  /**
+   * on-demand instances
+   */
+  ON_DEMAND = 'ON_DEMAND'
+}
+
 /**
  * The remote access (SSH) configuration to use with your node group.
  *
@@ -95,7 +109,7 @@ export interface NodegroupOptions {
   /**
    * The AMI type for your node group.
    *
-   * @default - auto-determined from the instanceType property.
+   * @default - auto-determined from the instanceTypes property.
    */
   readonly amiType?: NodegroupAmiType;
   /**
@@ -138,8 +152,15 @@ export interface NodegroupOptions {
    * `AL2_x86_64_GPU` with the amiType parameter.
    *
    * @default t3.medium
+   * @deprecated Use `instanceTypes` instead.
    */
   readonly instanceType?: InstanceType;
+  /**
+   * The instance types to use for your node group.
+   * @default t3.medium will be used according to the cloudformation document.
+   * @see - https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-instancetypes
+   */
+  readonly instanceTypes?: InstanceType[];
   /**
    * The Kubernetes labels to be applied to the nodes in the node group when they are created.
    *
@@ -183,6 +204,12 @@ export interface NodegroupOptions {
    * @default - no launch template
    */
   readonly launchTemplateSpec?: LaunchTemplateSpec;
+  /**
+   * The capacity type of the nodegroup.
+   *
+   * @default - ON_DEMAND
+   */
+  readonly capacityType?: CapacityType;
 }
 
 /**
@@ -199,6 +226,10 @@ export interface NodegroupProps extends NodegroupOptions {
  * The Nodegroup resource class
  */
 export class Nodegroup extends Resource implements INodegroup {
+  /**
+   * Default instanceTypes
+   */
+  public static readonly DEFAULT_INSTANCE_TYPES = [new InstanceType('t3.medium')];
   /**
    * Import the Nodegroup from attributes
    */
@@ -253,6 +284,25 @@ export class Nodegroup extends Resource implements INodegroup {
       throw new Error(`Minimum capacity ${this.minSize} can't be greater than desired size ${this.desiredSize}`);
     }
 
+    if (props.instanceType && props.instanceTypes) {
+      throw new Error('"instanceType is deprecated, please use "instanceTypes" only.');
+    }
+
+    if (props.instanceType) {
+      Annotations.of(this).addWarning('"instanceType" is deprecated and will be removed in the next major version. please use "instanceTypes" instead');
+    }
+    const instanceTypes = props.instanceTypes ?? (props.instanceType ? [props.instanceType] : Nodegroup.DEFAULT_INSTANCE_TYPES);
+    // get unique AMI types from instanceTypes
+    const uniqAmiTypes = getAmiTypes(instanceTypes);
+    // uniqAmiTypes.length should be at least 1
+    if (uniqAmiTypes.length > 1) {
+      throw new Error('instanceTypes of different CPU architectures is not allowed');
+    }
+    const determinedAmiType = uniqAmiTypes[0];
+    if (props.amiType && props.amiType !== determinedAmiType) {
+      throw new Error(`The specified AMI does not match the instance types architecture, either specify ${determinedAmiType} or dont specify any`);
+    }
+
     if (!props.nodeRole) {
       const ngRole = new Role(this, 'NodeGroupRole', {
         assumedBy: new ServicePrincipal('ec2.amazonaws.com'),
@@ -271,11 +321,12 @@ export class Nodegroup extends Resource implements INodegroup {
       nodegroupName: props.nodegroupName,
       nodeRole: this.role.roleArn,
       subnets: this.cluster.vpc.selectSubnets(props.subnets).subnetIds,
-      amiType: props.amiType ?? (props.instanceType ? getAmiTypeForInstanceType(props.instanceType).toString() :
-        undefined),
+      // AmyType is not allowed by CFN when specifying an image id in your launch template.
+      amiType: props.launchTemplateSpec === undefined ? determinedAmiType : undefined,
       diskSize: props.diskSize,
       forceUpdateEnabled: props.forceUpdate ?? true,
-      instanceTypes: props.instanceType ? [props.instanceType.toString()] : undefined,
+      instanceTypes: props.instanceTypes ? props.instanceTypes.map(t => t.toString()) :
+        props.instanceType ? [props.instanceType.toString()] : undefined,
       labels: props.labels,
       releaseVersion: props.releaseVersion,
       remoteAccess: props.remoteAccess ? {
@@ -291,17 +342,21 @@ export class Nodegroup extends Resource implements INodegroup {
       tags: props.tags,
     });
 
+    if (props.capacityType) {
+      resource.addPropertyOverride('CapacityType', props.capacityType.valueOf());
+    }
+
     if (props.launchTemplateSpec) {
       if (props.diskSize) {
         // see - https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html
         // and https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-disksize
         throw new Error('diskSize must be specified within the launch template');
       }
-      if (props.instanceType) {
-        // see - https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html
-        // and https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-disksize
-        throw new Error('Instance types must be specified within the launch template');
-      }
+      /**
+       * Instance types can be specified either in `instanceType` or launch template but not both. AS we can not check the content of
+       * the provided launch template and the `instanceType` property is preferrable. We allow users to define `instanceType` property here.
+       * see - https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-instancetypes
+       */
       // TODO: update this when the L1 resource spec is updated.
       resource.addPropertyOverride('LaunchTemplate', {
         Id: props.launchTemplateSpec.id,
@@ -340,3 +395,8 @@ function getAmiTypeForInstanceType(instanceType: InstanceType) {
           NodegroupAmiType.AL2_X86_64;
 }
 
+function getAmiTypes(instanceType: InstanceType[]) {
+  const amiTypes = instanceType.map(i =>getAmiTypeForInstanceType(i));
+  // retuen unique AMI types
+  return [...new Set(amiTypes)];
+}

packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json

@@ -1251,6 +1251,13 @@
                 ]
               },
               "\\\",\\\"username\\\":\\\"system:node:{{EC2PrivateDNSName}}\\\",\\\"groups\\\":[\\\"system:bootstrappers\\\",\\\"system:nodes\\\"]},{\\\"rolearn\\\":\\\"",
+              {
+                "Fn::GetAtt": [
+                  "ClusterNodegroupextrangspotNodeGroupRoleB53B4857",
+                  "Arn"
+                ]
+              },
+              "\\\",\\\"username\\\":\\\"system:node:{{EC2PrivateDNSName}}\\\",\\\"groups\\\":[\\\"system:bootstrappers\\\",\\\"system:nodes\\\"]},{\\\"rolearn\\\":\\\"",
               {
                 "Fn::GetAtt": [
                   "ClusterNodegroupextrangarmNodeGroupRoleADF5749F",
@@ -3251,6 +3258,109 @@
         }
       }
     },
+    "ClusterNodegroupextrangspotNodeGroupRoleB53B4857": {
+      "Type": "AWS::IAM::Role",
+      "Properties": {
+        "AssumeRolePolicyDocument": {
+          "Statement": [
+            {
+              "Action": "sts:AssumeRole",
+              "Effect": "Allow",
+              "Principal": {
+                "Service": {
+                  "Fn::Join": [
+                    "",
+                    [
+                      "ec2.",
+                      {
+                        "Ref": "AWS::URLSuffix"
+                      }
+                    ]
+                  ]
+                }
+              }
+            }
+          ],
+          "Version": "2012-10-17"
+        },
+        "ManagedPolicyArns": [
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:",
+                {
+                  "Ref": "AWS::Partition"
+                },
+                ":iam::aws:policy/AmazonEKSWorkerNodePolicy"
+              ]
+            ]
+          },
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:",
+                {
+                  "Ref": "AWS::Partition"
+                },
+                ":iam::aws:policy/AmazonEKS_CNI_Policy"
+              ]
+            ]
+          },
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:",
+                {
+                  "Ref": "AWS::Partition"
+                },
+                ":iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
+              ]
+            ]
+          }
+        ]
+      }
+    },
+    "ClusterNodegroupextrangspotB327AE6B": {
+      "Type": "AWS::EKS::Nodegroup",
+      "Properties": {
+        "ClusterName": {
+          "Ref": "Cluster9EE0221C"
+        },
+        "NodeRole": {
+          "Fn::GetAtt": [
+            "ClusterNodegroupextrangspotNodeGroupRoleB53B4857",
+            "Arn"
+          ]
+        },
+        "Subnets": [
+          {
+            "Ref": "VpcPrivateSubnet1Subnet536B997A"
+          },
+          {
+            "Ref": "VpcPrivateSubnet2Subnet3788AAA1"
+          },
+          {
+            "Ref": "VpcPrivateSubnet3SubnetF258B56E"
+          }
+        ],
+        "AmiType": "AL2_x86_64",
+        "ForceUpdateEnabled": true,
+        "InstanceTypes": [
+          "c5.large",
+          "c5a.large",
+          "c5d.large"
+        ],
+        "ScalingConfig": {
+          "DesiredSize": 3,
+          "MaxSize": 3,
+          "MinSize": 3
+        },
+        "CapacityType": "SPOT"
+      }
+    },
     "ClusterNodegroupextrangarmNodeGroupRoleADF5749F": {
       "Type": "AWS::IAM::Role",
       "Properties": {

packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts

@@ -53,6 +53,8 @@ class EksClusterStack extends TestStack {
 
     this.assertNodeGroupX86();
 
+    this.assertNodeGroupSpot();
+
     this.assertNodeGroupArm();
 
     this.assertNodeGroupCustomAmi();
@@ -162,6 +164,20 @@ class EksClusterStack extends TestStack {
       nodeRole: this.cluster.defaultCapacity ? this.cluster.defaultCapacity.role : undefined,
     });
   }
+  private assertNodeGroupSpot() {
+    // add a extra nodegroup
+    this.cluster.addNodegroupCapacity('extra-ng-spot', {
+      instanceTypes: [
+        new ec2.InstanceType('c5.large'),
+        new ec2.InstanceType('c5a.large'),
+        new ec2.InstanceType('c5d.large'),
+      ],
+      minSize: 3,
+      // reusing the default capacity nodegroup instance role when available
+      nodeRole: this.cluster.defaultCapacity ? this.cluster.defaultCapacity.role : undefined,
+      capacityType: eks.CapacityType.SPOT,
+    });
+  }
   private assertNodeGroupCustomAmi() {
     // add a extra nodegroup
     const userData = ec2.UserData.forLinux();