[WIP] Refactor OS/AMI flavor support adding more flexibility to configure launchtemplates

charts/karpenter/crds/karpenter.sh_provisioners.yaml

@@ -40,16 +40,97 @@ spec:
               Node properties are determined from a combination of provisioner and
               pod scheduling constraints.
             properties:
+              containerRuntimeConfiguration:
+                description: ContainerRuntimeConfiguration are options passed to the
+                  container runtime when provisioning nodes
+                properties:
+                  registryMirrors:
+                    description: RegistryMirrors a set of RegistryMirror configurations.
+                    items:
+                      description: RegistryMirror configuration
+                      properties:
+                        endpoints:
+                          description: Endpoints the endpoints to use as mirrors for
+                            that registry.
+                          items:
+                            description: RegistryMirrorEndpoint configuration.
+                            properties:
+                              url:
+                                description: URL of the registry mirror endpoint.
+                                type: string
+                            type: object
+                          type: array
+                        registry:
+                          description: Registry the registry's domain name or "*"
+                            to match all registries.
+                          type: string
+                      type: object
+                    type: array
+                type: object
               kubeletConfiguration:
                 description: KubeletConfiguration are options passed to the kubelet
-                  when provisioning nodes
+                  when provisioning nodes, this is a subset of https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration.
                 properties:
+                  allowedUnsafeSysctls:
+                    description: AllowedUnsafeSysctls a comma separated whitelist
+                      of unsafe sysctls or sysctl patterns (ending in `∗`).
+                    items:
+                      type: string
+                    type: array
                   clusterDNS:
-                    description: clusterDNS is a list of IP addresses for the cluster
+                    description: ClusterDNS is a list of IP addresses for the cluster
                       DNS server. Note that not all providers may use all addresses.
                     items:
                       type: string
                     type: array
+                  containerLogMaxFiles:
+                    description: ContainerLogMaxFiles specifies the maximum number
+                      of container log files that can be present for a container.
+                    format: int32
+                    type: integer
+                  containerLogMaxSize:
+                    description: ContainerLogMaxSize is a quantity defining the maximum
+                      size of the container log file before it is rotated.
+                    type: string
+                  eventBurst:
+                    description: EventBurst is the maximum size of a burst of event
+                      creations, temporarily allows event creations to burst to this
+                      number, while still not exceeding eventRecordQPS.
+                    format: int32
+                    type: integer
+                  eventRecordQPS:
+                    description: EventRecordQPS is the maximum event creations per
+                      second. If 0, there is no limit enforced.
+                    format: int32
+                    type: integer
+                  evictionHard:
+                    additionalProperties:
+                      type: string
+                    description: EvictionHard is a map of signal names to quantities
+                      that defines hard eviction thresholds.
+                    type: object
+                  kubeAPIBurst:
+                    description: KubeAPIBurst is the burst to allow while talking
+                      with kubernetes API server.
+                    format: int32
+                    type: integer
+                  kubeAPIQPS:
+                    description: KubeAPIQPS is the QPS to use while talking with kubernetes
+                      apiserver.
+                    format: int32
+                    type: integer
+                  registryBurst:
+                    description: RegistryBurst is the maximum size of bursty pulls,
+                      temporarily allows pulls to burst to this number, while still
+                      not exceeding registryPullQPS.
+                    format: int32
+                    type: integer
+                  registryPullQPS:
+                    description: RegistryPullQPS is the limit of registry pulls per
+                      second. The value must not be a negative number. Setting it
+                      to 0 means no limit.
+                    format: int32
+                    type: integer
                 type: object
               labels:
                 additionalProperties:

go.mod

@@ -14,7 +14,6 @@ require (
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.18.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/pelletier/go-toml/v2 v2.0.0-beta.5
 	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
 	go.uber.org/multierr v1.7.0
@@ -29,6 +28,8 @@ require (
 	sigs.k8s.io/controller-runtime v0.9.7
 )
 
+require k8s.io/utils v0.0.0-20210802155522-efc7438f0176
+
 require (
 	cloud.google.com/go v0.97.0 // indirect
 	contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect
@@ -68,6 +69,7 @@ require (
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/prometheus/statsd_exporter v0.21.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/automaxprocs v1.4.0 // indirect
@@ -91,7 +93,6 @@ require (
 	k8s.io/klog v1.0.0 // indirect
 	k8s.io/klog/v2 v2.8.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
-	k8s.io/utils v0.0.0-20210802155522-efc7438f0176 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )

go.sum

@@ -593,10 +593,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml/v2 v2.0.0-beta.5 h1:zRY68WYiDE9YZLflUWuyOny5YA+DwvyFzZBfb8E/9wk=
-github.com/pelletier/go-toml/v2 v2.0.0-beta.5/go.mod h1:ke6xncR3W76Ba8xnVxkrZG0js6Rd2BsQEAYrfgJ6eQA=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=

pkg/apis/provisioning/v1alpha5/constraints.go

@@ -34,9 +34,13 @@ type Constraints struct {
 	Taints Taints `json:"taints,omitempty"`
 	// Requirements are layered with Labels and applied to every node.
 	Requirements Requirements `json:"requirements,inline,omitempty"`
-	// KubeletConfiguration are options passed to the kubelet when provisioning nodes
+	// KubeletConfiguration are options passed to the kubelet when provisioning nodes, this
+	// is a subset of https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration.
 	//+optional
-	KubeletConfiguration *KubeletConfiguration `json:"kubeletConfiguration,omitempty"`
+	KubeletConfiguration KubeletConfiguration `json:"kubeletConfiguration,omitempty"`
+	// ContainerRuntimeConfiguration are options passed to the container runtime when provisioning nodes
+	//+optional
+	ContainerRuntimeConfiguration ContainerRuntimeConfiguration `json:"containerRuntimeConfiguration,omitempty"`
 	// Provider contains fields specific to your cloudprovider.
 	// +kubebuilder:pruning:PreserveUnknownFields
 	Provider *Provider `json:"provider,omitempty"`

pkg/apis/provisioning/v1alpha5/container_runtime_configuration.go

@@ -0,0 +1,63 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha5
+
+import (
+	"strings"
+
+	"knative.dev/pkg/apis"
+)
+
+type (
+	// ContainerRuntimeConfiguration defines args to be used when configuring the Container Runtime.
+	// Note, not all Providers or Container Runtime implementations will support all of these settings.
+	ContainerRuntimeConfiguration struct {
+		// RegistryMirrors a set of RegistryMirror configurations.
+		//+optional
+		RegistryMirrors []RegistryMirror `json:"registryMirrors,omitempty"`
+	}
+
+	// RegistryMirror configuration
+	RegistryMirror struct {
+		// Registry the registry's domain name or "*" to match all registries.
+		Registry string `json:"registry,omitempty"`
+		// Endpoints the endpoints to use as mirrors for that registry.
+		Endpoints []RegistryMirrorEndpoint `json:"endpoints,omitempty"`
+	}
+
+	// RegistryMirrorEndpoint configuration.
+	RegistryMirrorEndpoint struct {
+		// URL of the registry mirror endpoint.
+		URL string `json:"url,omitempty"`
+	}
+)
+
+func (c *ContainerRuntimeConfiguration) validate() (errs *apis.FieldError) {
+	return c.validateRegistryMirrors()
+}
+
+func (c *ContainerRuntimeConfiguration) validateRegistryMirrors() (errs *apis.FieldError) {
+	for key, mirror := range c.RegistryMirrors {
+		if len(strings.TrimSpace(mirror.Registry)) == 0 {
+			errs = errs.Also(errs, apis.ErrMissingField("registry")).ViaFieldIndex("registryMirrors", key)
+		}
+		for ekey, ep := range mirror.Endpoints {
+			if len(strings.TrimSpace(ep.URL)) == 0 {
+				errs = errs.Also(errs, apis.ErrMissingField("url")).ViaFieldIndex("registryMirrors", key).ViaFieldIndex("endpoints", ekey)
+			}
+		}
+	}
+	return errs
+}

pkg/apis/provisioning/v1alpha5/kubelet_configuration.go

@@ -14,12 +14,61 @@ limitations under the License.
 
 package v1alpha5
 
+import (
+	"knative.dev/pkg/apis"
+	"knative.dev/pkg/ptr"
+)
+
 // KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes.
 // They are a subset of the upstream types, recognizing not all options may be supported.
-// Wherever possible, the types and names should reflect the upstream kubelet types.
+// Wherever possible, the types and names should reflect the upstream kubelet types from
+// https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration
 type KubeletConfiguration struct {
-	// clusterDNS is a list of IP addresses for the cluster DNS server.
+	// ClusterDNS is a list of IP addresses for the cluster DNS server.
 	// Note that not all providers may use all addresses.
 	//+optional
 	ClusterDNS []string `json:"clusterDNS,omitempty"`
+	// EventRecordQPS is the maximum event creations per second. If 0,
+	// there is no limit enforced.
+	EventRecordQPS *int32 `json:"eventRecordQPS,omitempty"`
+	// EventBurst is the maximum size of a burst of event creations,
+	// temporarily allows event creations to burst to this number, while still not exceeding eventRecordQPS.
+	EventBurst *int32 `json:"eventBurst,omitempty"`
+	// RegistryPullQPS is the limit of registry pulls per second.
+	// The value must not be a negative number. Setting it to 0 means no limit.
+	RegistryPullQPS *int32 `json:"registryPullQPS,omitempty"`
+	// RegistryBurst is the maximum size of bursty pulls, temporarily allows pulls to burst to this number,
+	// while still not exceeding registryPullQPS.
+	RegistryBurst *int32 `json:"registryBurst,omitempty"`
+	// KubeAPIQPS is the QPS to use while talking with kubernetes apiserver.
+	KubeAPIQPS *int32 `json:"kubeAPIQPS,omitempty"`
+	// KubeAPIBurst is the burst to allow while talking with kubernetes API server.
+	KubeAPIBurst *int32 `json:"kubeAPIBurst,omitempty"`
+	// ContainerLogMaxSize is a quantity defining the maximum size of the container log file before it is rotated.
+	ContainerLogMaxSize *string `json:"containerLogMaxSize,omitempty"`
+	// ContainerLogMaxFiles specifies the maximum number of container log files that can be present for a container.
+	ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"`
+	// AllowedUnsafeSysctls a comma separated whitelist of unsafe sysctls or sysctl patterns (ending in `∗`).
+	AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"`
+	// EvictionHard is a map of signal names to quantities that defines hard eviction thresholds.
+	EvictionHard map[string]string `json:"evictionHard,omitempty"`
+}
+
+func (k *KubeletConfiguration) validate() (errs *apis.FieldError) {
+	return errs.Also(
+		addErrIfNegative(k.EventRecordQPS, "eventRecordQPS"),
+		addErrIfNegative(k.EventBurst, "eventBurst"),
+		addErrIfNegative(k.RegistryPullQPS, "registryPullQPS"),
+		addErrIfNegative(k.RegistryBurst, "registryBurst"),
+		addErrIfNegative(k.KubeAPIQPS, "kubeAPIQPS"),
+		addErrIfNegative(k.KubeAPIBurst, "kubeAPIBurst"),
+		addErrIfNegative(k.ContainerLogMaxFiles, "containerLogMaxFiles"),
+	)
+}
+
+func addErrIfNegative(num *int32, name string) (errs *apis.FieldError) {
+	if ptr.Int32Value(num) < 0 {
+		return apis.ErrInvalidValue("cannot be negative", name)
+	}
+	return errs
 }

pkg/apis/provisioning/v1alpha5/provisioner_validation.go

@@ -67,6 +67,8 @@ func (c *Constraints) Validate(ctx context.Context) (errs *apis.FieldError) {
 		c.validateLabels(),
 		c.validateTaints(),
 		c.validateRequirements(),
+		c.validateKubeletConfig(),
+		c.validateContainerRuntimeConfiguration(),
 		ValidateHook(ctx, c),
 	)
 }
@@ -155,3 +157,11 @@ func (c *Constraints) validateRequirements() (errs *apis.FieldError) {
 	}
 	return errs
 }
+
+func (c *Constraints) validateKubeletConfig() (errs *apis.FieldError) {
+	return c.KubeletConfiguration.validate().ViaField("kubeletConfiguration")
+}
+
+func (c *Constraints) validateContainerRuntimeConfiguration() (errs *apis.FieldError) {
+	return c.ContainerRuntimeConfiguration.validate().ViaField("containerRuntimeConfiguration")
+}