Merge pull request #736 from aws-quickstart/feature/karpenter

Karpenter Addon: adding AMI Selector, refactor to use NodeTemplate

docs/addons/karpenter.md

@@ -44,6 +44,9 @@ const karpenterAddonProps = {
     effect: "NoSchedule",
   }],
   amiFamily: "AL2",
+  amiSelector: {
+    "karpenter.sh/discovery/MyClusterName": '*',
+  },
   consolidation: { enabled: true },
   ttlSecondsUntilExpired: 2592000,
   weight: 20,
@@ -81,12 +84,12 @@ blueprints-addon-karpenter-54fd978b89-hclmp   2/2     Running   0          99m
 2. Creates `karpenter` namespace.
 3. Creates Kubernetes Service Account, and associate AWS IAM Role with Karpenter Controller Policy attached using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html).
 4. Deploys Karpenter helm chart in the `karpenter` namespace, configuring cluster name and cluster endpoint on the controller by default.
-5. (Optionally) provisions a default Karpenter Provisioner CRD based on user-provided [spec.requirements](https://karpenter.sh/v0.12.1/provisioner/#specrequirements), [AMI type](https://karpenter.sh/v0.12.1/aws/provisioning/#amazon-machine-image-ami-family), taints and tags. If created, the provisioner will discover the EKS VPC subnets and security groups to launch the nodes with.
+5. (Optionally) provisions a default Karpenter Provisioner and AWSNodeTemplate CRD based on user-provided parameters such as [spec.requirements](https://karpenter.sh/docs/concepts/provisioners/#specrequirements), [AMI type](https://karpenter.sh/v0.12.1/aws/provisioning/#amazon-machine-image-ami-family),[weight](https://karpenter.sh/docs/concepts/provisioners/#specweight), [Subnet Selector](https://karpenter.sh/docs/concepts/node-templates/#specsubnetselector), and [Security Group Selector](https://karpenter.sh/docs/concepts/node-templates/#specsecuritygroupselector). If created, the provisioner will discover the EKS VPC subnets and security groups to launch the nodes with.
 
 **NOTE:**
 1. The default provisioner is created only if both the subnet tags and the security group tags are provided.
-2. Provisioner spec requirement fields are not necessary, as karpenter will dynamically choose (i.e. leaving instance-type blank will let karpenter choose approrpriate sizing).
-3. Consolidation, which is a flag that enables , is supported on versions 0.15.0 and later. It is also mutually exclusive with `ttlSecondsAfterempty`, so if you provide both properties, the addon will throw an error.
+2. Provisioner spec requirement fields are not necessary, as karpenter will dynamically choose (i.e. leaving instance-type blank will let karpenter choose appropriate sizing).
+3. Consolidation, which is a flag that enables , is supported on versions 0.15.0 and later. It is also mutually exclusive with `ttlSecondsAfterEmpty`, so if you provide both properties, the addon will throw an error.
 4. Weight, which is a property to prioritize provisioners based on weight, is supported on versions 0.16.0 and later. Addon will throw an error if weight is provided for earlier versions.
 5. Interruption Handling, which is a native way to handle interruption due to involuntary interruption events, is supported on versions 0.19.0 and later. For interruption handling in the earlier versions, Karpenter supports using AWS Node Interruption Handler (which you will need to add as an add-on and ***must be in add-on array after the Karpenter add-on*** for it to work.
 

examples/blueprint-construct/index.ts

@@ -120,6 +120,9 @@ export default class BlueprintConstruct {
                     value: "test",
                     effect: "NoSchedule",
                 }],
+                amiSelector: {
+                    "karpenter.sh/discovery/MyClusterName": '*',
+                },
                 consolidation: { enabled: true },
                 ttlSecondsUntilExpired: 2592000,
                 weight: 20,

lib/addons/karpenter/index.ts

@@ -1,7 +1,7 @@
 import * as iam from 'aws-cdk-lib/aws-iam';
 import { Construct } from "constructs";
 import merge from 'ts-deepmerge';
-import { ClusterInfo } from '../../spi';
+import { ClusterInfo, Values, Taint } from '../../spi';
 import { conflictsWith, createNamespace, createServiceAccount, setPath, } from '../../utils';
 import { HelmAddOn, HelmAddOnProps, HelmAddOnUserProps } from '../helm-addon';
 import { KarpenterControllerPolicy } from './iam';
@@ -19,6 +19,28 @@ import { Cluster } from 'aws-cdk-lib/aws-eks';
  * Configuration options for the add-on
  */
 interface KarpenterAddOnProps extends HelmAddOnUserProps {
+    /**
+     * Taints for the provisioned nodes - Taints may prevent pods from scheduling if they are not tolerated by the pod.
+     */
+    taints?: Taint[],
+
+    /**
+     * Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this\
+     * provisioner. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for
+     * removing the taint after it has finished initializing the node.
+     */
+    startupTaints?: Taint[],
+
+    /**
+     * Labels applied to all nodes
+     */
+    labels?: Values,
+
+    /**
+     * Annotations applied to all nodes
+     */
+    annotations?: Values,
+
     /**
      * Requirement properties for a Provisioner (Optional) - If not provided, the add-on will
      * deploy a Provisioner with default values.
@@ -29,31 +51,26 @@ interface KarpenterAddOnProps extends HelmAddOnUserProps {
         vals: string[],
     }[]
 
-    taints?: {
-        key: string,
-        value: string,
-        effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute",
-    }[]
-
     /**
      * Tags needed for subnets - Subnet tags and security group tags are required for the provisioner to be created
      */
-    subnetTags?: {
-        [key: string]: string
-    }
+    subnetTags?: Values,
 
     /**
      * Tags needed for security groups - Subnet tags and security group tags are required for the provisioner to be created
      */
-    securityGroupTags?: {
-        [key: string]: string
-    }
+    securityGroupTags?: Values,
 
     /**
      * AMI Family: If provided, Karpenter will automatically query the appropriate EKS optimized AMI via AWS Systems Manager
      */
     amiFamily?: "AL2" | "Bottlerocket" | "Ubuntu"
 
+    /**
+     * AMI Selector
+     */
+    amiSelector?: Values,
+
     /**
      * Enables consolidation which attempts to reduce cluster cost by both removing un-needed nodes and down-sizing those that can't be removed.  
      * Mutually exclusive with the ttlSecondsAfterEmpty parameter.
@@ -117,7 +134,7 @@ const RELEASE = 'blueprints-addon-karpenter';
 const defaultProps: HelmAddOnProps = {
     name: KARPENTER,
     namespace: KARPENTER,
-    version: 'v0.27.3',
+    version: 'v0.28.0',
     chart: KARPENTER,
     release: KARPENTER,
     repository: 'oci://public.ecr.aws/karpenter/karpenter',
@@ -151,7 +168,11 @@ export class KarpenterAddOn extends HelmAddOn {
         const subnetTags = this.options.subnetTags || {};
         const sgTags = this.options.securityGroupTags || {};
         const taints = this.options.taints || [];
+        const startupTaints = this.options.startupTaints || [];
+        const labels = this.options.labels || {};
+        const annotations = this.options.annotations || {};
         const amiFamily = this.options.amiFamily;
+        const amiSelector = this.options.amiSelector;
         const ttlSecondsAfterEmpty = this.options.ttlSecondsAfterEmpty || null;
         const ttlSecondsUntilExpired = this.options.ttlSecondsUntilExpired || null;
         const weight = this.options.weight || null;
@@ -280,21 +301,37 @@ export class KarpenterAddOn extends HelmAddOn {
                 kind: 'Provisioner',
                 metadata: { name: 'default' },
                 spec: {
-                    consolidation: consolidation,
-                    requirements: this.convert(requirements),
+                    providerRef: {
+                        name: "default"
+                    },
                     taints: taints,
+                    startupTaints: startupTaints,
+                    labels: labels,
+                    annotations: annotations,
+                    requirements: this.convert(requirements),
                     limits: limits,
-                    provider: {
-                        amiFamily: amiFamily,
-                        subnetSelector: subnetTags,
-                        securityGroupSelector: sgTags,
-                    },
+                    consolidation: consolidation,
                     ttlSecondsUntilExpired: ttlSecondsUntilExpired,
                     ttlSecondsAfterEmpty: ttlSecondsAfterEmpty,
                     weight: weight,
                 },
             });
             provisioner.node.addDependency(karpenterChart);
+
+            const nodeTemplate = cluster.addManifest('default-node-template', {
+                apiVersion: "karpenter.k8s.aws/v1alpha1",
+                kind: "AWSNodeTemplate",
+                metadata: {
+                    name: "default"
+                },
+                spec: {
+                    amiFamily: amiFamily,
+                    amiSelector: amiSelector,
+                    subnetSelector: subnetTags,
+                    securityGroupSelector: sgTags,
+                },
+            });
+            nodeTemplate.node.addDependency(provisioner);
         }
 
         return Promise.resolve(karpenterChart);

lib/spi/types.ts

@@ -24,6 +24,15 @@ export type Values = {
     [key: string]: any;
 };
 
+/**
+ * Utility type for Kubernetes taints passed to Helm or GitOps applications.
+ */
+export type Taint = {
+    key: string,
+    value: string,
+    effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute",
+};
+
 /**
  * Interface that includes a reference to a Git repository for reuse, without credentials
  * and other access information.