diff --git a/docs/addons/karpenter.md b/docs/addons/karpenter.md index de226a821..0a08c5502 100644 --- a/docs/addons/karpenter.md +++ b/docs/addons/karpenter.md @@ -44,6 +44,9 @@ const karpenterAddonProps = { effect: "NoSchedule", }], amiFamily: "AL2", + amiSelector: { + "karpenter.sh/discovery/MyClusterName": '*', + }, consolidation: { enabled: true }, ttlSecondsUntilExpired: 2592000, weight: 20, @@ -81,12 +84,12 @@ blueprints-addon-karpenter-54fd978b89-hclmp 2/2 Running 0 99m 2. Creates `karpenter` namespace. 3. Creates Kubernetes Service Account, and associate AWS IAM Role with Karpenter Controller Policy attached using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html). 4. Deploys Karpenter helm chart in the `karpenter` namespace, configuring cluster name and cluster endpoint on the controller by default. -5. (Optionally) provisions a default Karpenter Provisioner CRD based on user-provided [spec.requirements](https://karpenter.sh/v0.12.1/provisioner/#specrequirements), [AMI type](https://karpenter.sh/v0.12.1/aws/provisioning/#amazon-machine-image-ami-family), taints and tags. If created, the provisioner will discover the EKS VPC subnets and security groups to launch the nodes with. +5. (Optionally) provisions a default Karpenter Provisioner and AWSNodeTemplate CRD based on user-provided parameters such as [spec.requirements](https://karpenter.sh/docs/concepts/provisioners/#specrequirements), [AMI type](https://karpenter.sh/v0.12.1/aws/provisioning/#amazon-machine-image-ami-family),[weight](https://karpenter.sh/docs/concepts/provisioners/#specweight), [Subnet Selector](https://karpenter.sh/docs/concepts/node-templates/#specsubnetselector), and [Security Group Selector](https://karpenter.sh/docs/concepts/node-templates/#specsecuritygroupselector). If created, the provisioner will discover the EKS VPC subnets and security groups to launch the nodes with. **NOTE:** 1. The default provisioner is created only if both the subnet tags and the security group tags are provided. -2. Provisioner spec requirement fields are not necessary, as karpenter will dynamically choose (i.e. leaving instance-type blank will let karpenter choose approrpriate sizing). -3. Consolidation, which is a flag that enables , is supported on versions 0.15.0 and later. It is also mutually exclusive with `ttlSecondsAfterempty`, so if you provide both properties, the addon will throw an error. +2. Provisioner spec requirement fields are not necessary, as karpenter will dynamically choose (i.e. leaving instance-type blank will let karpenter choose appropriate sizing). +3. Consolidation, which is a flag that enables , is supported on versions 0.15.0 and later. It is also mutually exclusive with `ttlSecondsAfterEmpty`, so if you provide both properties, the addon will throw an error. 4. Weight, which is a property to prioritize provisioners based on weight, is supported on versions 0.16.0 and later. Addon will throw an error if weight is provided for earlier versions. 5. Interruption Handling, which is a native way to handle interruption due to involuntary interruption events, is supported on versions 0.19.0 and later. For interruption handling in the earlier versions, Karpenter supports using AWS Node Interruption Handler (which you will need to add as an add-on and ***must be in add-on array after the Karpenter add-on*** for it to work. diff --git a/examples/blueprint-construct/index.ts b/examples/blueprint-construct/index.ts index 4595f40db..513be4169 100644 --- a/examples/blueprint-construct/index.ts +++ b/examples/blueprint-construct/index.ts @@ -120,6 +120,9 @@ export default class BlueprintConstruct { value: "test", effect: "NoSchedule", }], + amiSelector: { + "karpenter.sh/discovery/MyClusterName": '*', + }, consolidation: { enabled: true }, ttlSecondsUntilExpired: 2592000, weight: 20, diff --git a/lib/addons/karpenter/index.ts b/lib/addons/karpenter/index.ts index 84699aab7..f692eacc2 100644 --- a/lib/addons/karpenter/index.ts +++ b/lib/addons/karpenter/index.ts @@ -1,7 +1,7 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import { Construct } from "constructs"; import merge from 'ts-deepmerge'; -import { ClusterInfo } from '../../spi'; +import { ClusterInfo, Values, Taint } from '../../spi'; import { conflictsWith, createNamespace, createServiceAccount, setPath, } from '../../utils'; import { HelmAddOn, HelmAddOnProps, HelmAddOnUserProps } from '../helm-addon'; import { KarpenterControllerPolicy } from './iam'; @@ -19,6 +19,28 @@ import { Cluster } from 'aws-cdk-lib/aws-eks'; * Configuration options for the add-on */ interface KarpenterAddOnProps extends HelmAddOnUserProps { + /** + * Taints for the provisioned nodes - Taints may prevent pods from scheduling if they are not tolerated by the pod. + */ + taints?: Taint[], + + /** + * Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this\ + * provisioner. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for + * removing the taint after it has finished initializing the node. + */ + startupTaints?: Taint[], + + /** + * Labels applied to all nodes + */ + labels?: Values, + + /** + * Annotations applied to all nodes + */ + annotations?: Values, + /** * Requirement properties for a Provisioner (Optional) - If not provided, the add-on will * deploy a Provisioner with default values. @@ -29,31 +51,26 @@ interface KarpenterAddOnProps extends HelmAddOnUserProps { vals: string[], }[] - taints?: { - key: string, - value: string, - effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute", - }[] - /** * Tags needed for subnets - Subnet tags and security group tags are required for the provisioner to be created */ - subnetTags?: { - [key: string]: string - } + subnetTags?: Values, /** * Tags needed for security groups - Subnet tags and security group tags are required for the provisioner to be created */ - securityGroupTags?: { - [key: string]: string - } + securityGroupTags?: Values, /** * AMI Family: If provided, Karpenter will automatically query the appropriate EKS optimized AMI via AWS Systems Manager */ amiFamily?: "AL2" | "Bottlerocket" | "Ubuntu" + /** + * AMI Selector + */ + amiSelector?: Values, + /** * Enables consolidation which attempts to reduce cluster cost by both removing un-needed nodes and down-sizing those that can't be removed. * Mutually exclusive with the ttlSecondsAfterEmpty parameter. @@ -117,7 +134,7 @@ const RELEASE = 'blueprints-addon-karpenter'; const defaultProps: HelmAddOnProps = { name: KARPENTER, namespace: KARPENTER, - version: 'v0.27.3', + version: 'v0.28.0', chart: KARPENTER, release: KARPENTER, repository: 'oci://public.ecr.aws/karpenter/karpenter', @@ -151,7 +168,11 @@ export class KarpenterAddOn extends HelmAddOn { const subnetTags = this.options.subnetTags || {}; const sgTags = this.options.securityGroupTags || {}; const taints = this.options.taints || []; + const startupTaints = this.options.startupTaints || []; + const labels = this.options.labels || {}; + const annotations = this.options.annotations || {}; const amiFamily = this.options.amiFamily; + const amiSelector = this.options.amiSelector; const ttlSecondsAfterEmpty = this.options.ttlSecondsAfterEmpty || null; const ttlSecondsUntilExpired = this.options.ttlSecondsUntilExpired || null; const weight = this.options.weight || null; @@ -280,21 +301,37 @@ export class KarpenterAddOn extends HelmAddOn { kind: 'Provisioner', metadata: { name: 'default' }, spec: { - consolidation: consolidation, - requirements: this.convert(requirements), + providerRef: { + name: "default" + }, taints: taints, + startupTaints: startupTaints, + labels: labels, + annotations: annotations, + requirements: this.convert(requirements), limits: limits, - provider: { - amiFamily: amiFamily, - subnetSelector: subnetTags, - securityGroupSelector: sgTags, - }, + consolidation: consolidation, ttlSecondsUntilExpired: ttlSecondsUntilExpired, ttlSecondsAfterEmpty: ttlSecondsAfterEmpty, weight: weight, }, }); provisioner.node.addDependency(karpenterChart); + + const nodeTemplate = cluster.addManifest('default-node-template', { + apiVersion: "karpenter.k8s.aws/v1alpha1", + kind: "AWSNodeTemplate", + metadata: { + name: "default" + }, + spec: { + amiFamily: amiFamily, + amiSelector: amiSelector, + subnetSelector: subnetTags, + securityGroupSelector: sgTags, + }, + }); + nodeTemplate.node.addDependency(provisioner); } return Promise.resolve(karpenterChart); diff --git a/lib/spi/types.ts b/lib/spi/types.ts index 328c7f985..6456405fb 100644 --- a/lib/spi/types.ts +++ b/lib/spi/types.ts @@ -24,6 +24,15 @@ export type Values = { [key: string]: any; }; +/** + * Utility type for Kubernetes taints passed to Helm or GitOps applications. + */ +export type Taint = { + key: string, + value: string, + effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute", +}; + /** * Interface that includes a reference to a Git repository for reuse, without credentials * and other access information.