cleanup

pkg/cloudprovider/aws/cloudprovider.go

@@ -26,10 +26,12 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ssm"
+	"github.com/patrickmn/go-cache"
 
 	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
 	"github.com/aws/karpenter/pkg/cloudprovider"
 	"github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1"
+	"github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver"
 	"github.com/aws/karpenter/pkg/utils/functional"
 	"github.com/aws/karpenter/pkg/utils/injection"
 	"github.com/aws/karpenter/pkg/utils/project"
@@ -87,7 +89,8 @@ func NewCloudProvider(ctx context.Context, options cloudprovider.Options) *Cloud
 			NewLaunchTemplateProvider(
 				ctx,
 				ec2api,
-				NewAMIProvider(ssm.New(sess), options.ClientSet),
+				options.ClientSet,
+				ltresolver.New(ssm.New(sess), cache.New(CacheTTL, CacheCleanupInterval)),
 				NewSecurityGroupProvider(ec2api),
 				getCABundle(ctx),
 			),

pkg/cloudprovider/aws/instance.go

@@ -54,6 +54,15 @@ type InstanceProvider struct {
 	launchTemplateProvider *LaunchTemplateProvider
 }
 
+func NewInstanceProvider(ec2api ec2iface.EC2API, instanceTypeProvider *InstanceTypeProvider, subnetProvider *SubnetProvider, launchTemplateProvider *LaunchTemplateProvider) *InstanceProvider {
+	return &InstanceProvider{
+		ec2api:                 ec2api,
+		instanceTypeProvider:   instanceTypeProvider,
+		subnetProvider:         subnetProvider,
+		launchTemplateProvider: launchTemplateProvider,
+	}
+}
+
 // Create an instance given the constraints.
 // instanceTypes should be sorted by priority for spot capacity type.
 // If spot is not used, the instanceTypes are not required to be sorted

pkg/cloudprovider/aws/instancetypes.go

@@ -95,6 +95,11 @@ func (p *InstanceTypeProvider) Get(ctx context.Context, provider *v1alpha1.AWS)
 	return result, nil
 }
 
+func (p *InstanceTypeProvider) Flush() {
+	p.cache.Flush()
+	p.unavailableOfferings.Flush()
+}
+
 func (p *InstanceTypeProvider) createOfferings(instanceType *InstanceType, subnetZones sets.String, availableZones sets.String) []cloudprovider.Offering {
 	offerings := []cloudprovider.Offering{}
 	for zone := range subnetZones.Intersection(availableZones) {
@@ -133,8 +138,8 @@ func (p *InstanceTypeProvider) getInstanceTypeZones(ctx context.Context) (map[st
 
 // getInstanceTypes retrieves all instance types from the ec2 DescribeInstanceTypes API using some opinionated filters
 func (p *InstanceTypeProvider) getInstanceTypes(ctx context.Context) (map[string]*InstanceType, error) {
-	if Cached, ok := p.cache.Get(InstanceTypesCacheKey); ok {
-		return Cached.(map[string]*InstanceType), nil
+	if cached, ok := p.cache.Get(InstanceTypesCacheKey); ok {
+		return cached.(map[string]*InstanceType), nil
 	}
 	instanceTypes := map[string]*InstanceType{}
 	if err := p.ec2api.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{

pkg/cloudprovider/aws/launchtemplate.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -28,35 +29,39 @@ import (
 	"github.com/patrickmn/go-cache"
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/client-go/kubernetes"
 	"knative.dev/pkg/logging"
 	"knative.dev/pkg/ptr"
 
 	"github.com/aws/karpenter/pkg/cloudprovider"
 	"github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1"
-	"github.com/aws/karpenter/pkg/cloudprovider/aws/launchtemplate"
+	"github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver"
 	"github.com/aws/karpenter/pkg/utils/functional"
 	"github.com/aws/karpenter/pkg/utils/injection"
 )
 
 const (
-	launchTemplateNameFormat = "Karpenter-%s-%s"
+	launchTemplateNameFormat  = "Karpenter-%s-%s"
+	kubernetesVersionCacheKey = "kubernetesVersion"
 )
 
 type LaunchTemplateProvider struct {
 	sync.Mutex
 	ec2api                ec2iface.EC2API
-	amiProvider           *AMIProvider
+	clientSet             *kubernetes.Clientset
+	ltResolver            *ltresolver.Resolver
 	securityGroupProvider *SecurityGroupProvider
 	cache                 *cache.Cache
 	logger                *zap.SugaredLogger
 	caBundle              *string
 }
 
-func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, amiProvider *AMIProvider, securityGroupProvider *SecurityGroupProvider, caBundle *string) *LaunchTemplateProvider {
+func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, clientSet *kubernetes.Clientset, ltResolver *ltresolver.Resolver, securityGroupProvider *SecurityGroupProvider, caBundle *string) *LaunchTemplateProvider {
 	l := &LaunchTemplateProvider{
 		ec2api:                ec2api,
+		clientSet:             clientSet,
 		logger:                logging.FromContext(ctx).Named("launchtemplate"),
-		amiProvider:           amiProvider,
+		ltResolver:            ltResolver,
 		securityGroupProvider: securityGroupProvider,
 		cache:                 cache.New(CacheTTL, CacheCleanupInterval),
 		caBundle:              caBundle,
@@ -66,7 +71,7 @@ func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, amiP
 	return l
 }
 
-func launchTemplateName(options *launchtemplate.Resolved) string {
+func launchTemplateName(options *ltresolver.ResolvedTemplate) string {
 	hash, err := hashstructure.Hash(options, hashstructure.FormatV2, nil)
 	if err != nil {
 		panic(fmt.Sprintf("hashing launch template, %s", err))
@@ -88,35 +93,30 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1.
 	if err != nil {
 		return nil, err
 	}
-	// Get constrained AMI ID
-	amis, err := p.amiProvider.Get(ctx, constraints, instanceTypes, launchtemplate.GetAMIFamily(constraints.AMIFamily, nil))
-	if err != nil {
-		return nil, err
-	}
-	// Construct launch templates
+	resolvedLaunchTemplates := p.ltResolver.Resolve(constraints, instanceTypes, &ltresolver.Options{
+		ClusterName:             injection.GetOptions(ctx).ClusterName,
+		ClusterEndpoint:         injection.GetOptions(ctx).ClusterEndpoint,
+		AWSENILimitedPodDensity: injection.GetOptions(ctx).AWSENILimitedPodDensity,
+		InstanceProfile:         instanceProfile,
+		SecurityGroupsIDs:       securityGroupsIDs,
+		Tags:                    constraints.Tags,
+		Labels:                  functional.UnionStringMaps(constraints.Labels, additionalLabels),
+		CABundle:                p.caBundle,
+	})
+
 	launchTemplates := map[string][]cloudprovider.InstanceType{}
-	for amiID, instanceTypes := range amis {
+	for _, resolvedLaunchTemplate := range resolvedLaunchTemplates {
 		// Ensure the launch template exists, or create it
-		launchTemplate, err := p.ensureLaunchTemplate(ctx, launchtemplate.Get(constraints, instanceTypes, &launchtemplate.Options{
-			ClusterName:             injection.GetOptions(ctx).ClusterName,
-			ClusterEndpoint:         injection.GetOptions(ctx).ClusterEndpoint,
-			AWSENILimitedPodDensity: injection.GetOptions(ctx).AWSENILimitedPodDensity,
-			InstanceProfile:         instanceProfile,
-			AMIID:                   amiID,
-			SecurityGroupsIDs:       securityGroupsIDs,
-			Tags:                    constraints.Tags,
-			Labels:                  functional.UnionStringMaps(constraints.Labels, additionalLabels),
-			CABundle:                p.caBundle,
-		}))
+		ec2LaunchTemplate, err := p.ensureLaunchTemplate(ctx, resolvedLaunchTemplate)
 		if err != nil {
 			return nil, err
 		}
-		launchTemplates[aws.StringValue(launchTemplate.LaunchTemplateName)] = instanceTypes
+		launchTemplates[*ec2LaunchTemplate.LaunchTemplateName] = resolvedLaunchTemplate.InstanceTypes
 	}
 	return launchTemplates, nil
 }
 
-func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, options *launchtemplate.Resolved) (*ec2.LaunchTemplate, error) {
+func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, options *ltresolver.ResolvedTemplate) (*ec2.LaunchTemplate, error) {
 	// Ensure that multiple threads don't attempt to create the same launch template
 	p.Lock()
 	defer p.Unlock()
@@ -150,7 +150,7 @@ func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, optio
 	return launchTemplate, nil
 }
 
-func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, options *launchtemplate.Resolved) (*ec2.LaunchTemplate, error) {
+func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, options *ltresolver.ResolvedTemplate) (*ec2.LaunchTemplate, error) {
 	output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{
 		LaunchTemplateName: aws.String(launchTemplateName(options)),
 		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
@@ -241,3 +241,17 @@ func (p *LaunchTemplateProvider) getInstanceProfile(ctx context.Context, constra
 	}
 	return defaultProfile, nil
 }
+
+func (p *LaunchTemplateProvider) kubeServerVersion(ctx context.Context) (string, error) {
+	if version, ok := p.cache.Get(kubernetesVersionCacheKey); ok {
+		return version.(string), nil
+	}
+	serverVersion, err := p.clientSet.Discovery().ServerVersion()
+	if err != nil {
+		return "", err
+	}
+	version := fmt.Sprintf("%s.%s", serverVersion.Major, strings.TrimSuffix(serverVersion.Minor, "+"))
+	p.cache.SetDefault(kubernetesVersionCacheKey, version)
+	logging.FromContext(ctx).Debugf("Discovered kubernetes version %s", version)
+	return version, nil
+}

pkg/cloudprovider/aws/launchtemplate/al2.go → pkg/cloudprovider/aws/ltresolver/al2.go

@@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package launchtemplate
+package ltresolver
 
 import (
 	"fmt"
@@ -23,7 +23,7 @@ import (
 	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
 	"github.com/aws/karpenter/pkg/cloudprovider"
 	"github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1"
-	"github.com/aws/karpenter/pkg/cloudprovider/aws/launchtemplate/bootstrap"
+	"github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver/bootstrap"
 )
 
 type AL2 struct {

pkg/cloudprovider/aws/ltresolver/ami.go

@@ -0,0 +1,52 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ltresolver
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ssm"
+	"github.com/aws/aws-sdk-go/service/ssm/ssmiface"
+	"github.com/patrickmn/go-cache"
+	"knative.dev/pkg/logging"
+
+	"github.com/aws/karpenter/pkg/cloudprovider"
+)
+
+type AMIProvider struct {
+	cache *cache.Cache
+	ssm   ssmiface.SSMAPI
+}
+
+// Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc
+func (p *AMIProvider) Get(ctx context.Context, instanceType cloudprovider.InstanceType, ssmQuery string) (string, error) {
+	if id, ok := p.cache.Get(ssmQuery); ok {
+		return id.(string), nil
+	}
+	output, err := p.ssm.GetParameterWithContext(ctx, &ssm.GetParameterInput{Name: aws.String(ssmQuery)})
+	if err != nil {
+		return "", fmt.Errorf("getting ssm parameter, %w", err)
+	}
+	ami := aws.StringValue(output.Parameter.Value)
+	p.cache.SetDefault(ssmQuery, ami)
+	logging.FromContext(ctx).Debugf("Discovered %s for query %s", ami, ssmQuery)
+	return ami, nil
+}
+
+func (p *AMIProvider) Flush() {
+	p.cache.Flush()
+}

pkg/cloudprovider/aws/launchtemplate/bottlerocket.go → pkg/cloudprovider/aws/ltresolver/bottlerocket.go

@@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package launchtemplate
+package ltresolver
 
 import (
 	"fmt"
@@ -24,7 +24,7 @@ import (
 	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
 	"github.com/aws/karpenter/pkg/cloudprovider"
 	"github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1"
-	"github.com/aws/karpenter/pkg/cloudprovider/aws/launchtemplate/bootstrap"
+	"github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver/bootstrap"
 )
 
 type Bottlerocket struct {