From 7d4e067bdb0f6cf0a1e075f8d48e79e5bca7d359 Mon Sep 17 00:00:00 2001 From: Brandon Date: Wed, 9 Mar 2022 13:10:47 -0600 Subject: [PATCH] cleanup --- pkg/cloudprovider/aws/ami.go | 100 ------------------ pkg/cloudprovider/aws/cloudprovider.go | 5 +- pkg/cloudprovider/aws/instance.go | 9 ++ pkg/cloudprovider/aws/instancetypes.go | 9 +- pkg/cloudprovider/aws/launchtemplate.go | 68 +++++++----- .../aws/{launchtemplate => ltresolver}/al2.go | 4 +- pkg/cloudprovider/aws/ltresolver/ami.go | 52 +++++++++ .../bootstrap/bootstrap.go | 0 .../bootstrap/bottlerocket.go | 0 .../bootstrap/eksbootstrap.go | 0 .../bottlerocket.go | 4 +- .../resolver.go} | 72 +++++++++---- .../{launchtemplate => ltresolver}/ubuntu.go | 4 +- pkg/cloudprovider/aws/securitygroups.go | 4 + pkg/cloudprovider/aws/subnets.go | 12 ++- pkg/cloudprovider/aws/suite_test.go | 6 +- 16 files changed, 190 insertions(+), 159 deletions(-) delete mode 100644 pkg/cloudprovider/aws/ami.go rename pkg/cloudprovider/aws/{launchtemplate => ltresolver}/al2.go (96%) create mode 100644 pkg/cloudprovider/aws/ltresolver/ami.go rename pkg/cloudprovider/aws/{launchtemplate => ltresolver}/bootstrap/bootstrap.go (100%) rename pkg/cloudprovider/aws/{launchtemplate => ltresolver}/bootstrap/bottlerocket.go (100%) rename pkg/cloudprovider/aws/{launchtemplate => ltresolver}/bootstrap/eksbootstrap.go (100%) rename pkg/cloudprovider/aws/{launchtemplate => ltresolver}/bottlerocket.go (96%) rename pkg/cloudprovider/aws/{launchtemplate/launchtemplate.go => ltresolver/resolver.go} (50%) rename pkg/cloudprovider/aws/{launchtemplate => ltresolver}/ubuntu.go (95%) diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go deleted file mode 100644 index debed78f5343..000000000000 --- a/pkg/cloudprovider/aws/ami.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package aws - -import ( - "context" - "fmt" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ssm" - "github.com/aws/aws-sdk-go/service/ssm/ssmiface" - "github.com/patrickmn/go-cache" - "k8s.io/client-go/kubernetes" - "knative.dev/pkg/logging" - - "github.com/aws/karpenter/pkg/cloudprovider" - "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" - "github.com/aws/karpenter/pkg/cloudprovider/aws/launchtemplate" -) - -const kubernetesVersionCacheKey = "kubernetesVersion" - -type AMIProvider struct { - cache *cache.Cache - ssm ssmiface.SSMAPI - clientSet *kubernetes.Clientset -} - -func NewAMIProvider(ssm ssmiface.SSMAPI, clientSet *kubernetes.Clientset) *AMIProvider { - return &AMIProvider{ - ssm: ssm, - clientSet: clientSet, - cache: cache.New(CacheTTL, CacheCleanupInterval), - } -} - -// Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc -func (p *AMIProvider) Get(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, amiFamily launchtemplate.AMIFamily) (map[string][]cloudprovider.InstanceType, error) { - version, err := p.kubeServerVersion(ctx) - if err != nil { - return nil, fmt.Errorf("kube server version, %w", err) - } - // Separate instance types by unique queries - amiQueries := map[string][]cloudprovider.InstanceType{} - for _, instanceType := range instanceTypes { - query := amiFamily.SSMAlias(version, instanceType) - amiQueries[query] = append(amiQueries[query], instanceType) - } - // Separate instance types by unique AMIIDs - amiIDs := map[string][]cloudprovider.InstanceType{} - for query, instanceTypes := range amiQueries { - amiID, err := p.getAMIID(ctx, query) - if err != nil { - return nil, err - } - amiIDs[amiID] = instanceTypes - } - return amiIDs, nil -} - -func (p *AMIProvider) kubeServerVersion(ctx context.Context) (string, error) { - if version, ok := p.cache.Get(kubernetesVersionCacheKey); ok { - return version.(string), nil - } - serverVersion, err := p.clientSet.Discovery().ServerVersion() - if err != nil { - return "", err - } - version := fmt.Sprintf("%s.%s", serverVersion.Major, strings.TrimSuffix(serverVersion.Minor, "+")) - p.cache.SetDefault(kubernetesVersionCacheKey, version) - logging.FromContext(ctx).Debugf("Discovered kubernetes version %s", version) - return version, nil -} - -func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error) { - if id, ok := p.cache.Get(query); ok { - return id.(string), nil - } - output, err := p.ssm.GetParameterWithContext(ctx, &ssm.GetParameterInput{Name: aws.String(query)}) - if err != nil { - return "", fmt.Errorf("getting ssm parameter, %w", err) - } - ami := aws.StringValue(output.Parameter.Value) - p.cache.SetDefault(query, ami) - logging.FromContext(ctx).Debugf("Discovered %s for query %s", ami, query) - return ami, nil -} diff --git a/pkg/cloudprovider/aws/cloudprovider.go b/pkg/cloudprovider/aws/cloudprovider.go index ecf4c13ccc92..77464c28a957 100644 --- a/pkg/cloudprovider/aws/cloudprovider.go +++ b/pkg/cloudprovider/aws/cloudprovider.go @@ -26,10 +26,12 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ssm" + "github.com/patrickmn/go-cache" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" + "github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver" "github.com/aws/karpenter/pkg/utils/functional" "github.com/aws/karpenter/pkg/utils/injection" "github.com/aws/karpenter/pkg/utils/project" @@ -87,7 +89,8 @@ func NewCloudProvider(ctx context.Context, options cloudprovider.Options) *Cloud NewLaunchTemplateProvider( ctx, ec2api, - NewAMIProvider(ssm.New(sess), options.ClientSet), + options.ClientSet, + ltresolver.New(ssm.New(sess), cache.New(CacheTTL, CacheCleanupInterval)), NewSecurityGroupProvider(ec2api), getCABundle(ctx), ), diff --git a/pkg/cloudprovider/aws/instance.go b/pkg/cloudprovider/aws/instance.go index 5d2d7914f254..a6101b8fe909 100644 --- a/pkg/cloudprovider/aws/instance.go +++ b/pkg/cloudprovider/aws/instance.go @@ -54,6 +54,15 @@ type InstanceProvider struct { launchTemplateProvider *LaunchTemplateProvider } +func NewInstanceProvider(ec2api ec2iface.EC2API, instanceTypeProvider *InstanceTypeProvider, subnetProvider *SubnetProvider, launchTemplateProvider *LaunchTemplateProvider) *InstanceProvider { + return &InstanceProvider{ + ec2api: ec2api, + instanceTypeProvider: instanceTypeProvider, + subnetProvider: subnetProvider, + launchTemplateProvider: launchTemplateProvider, + } +} + // Create an instance given the constraints. // instanceTypes should be sorted by priority for spot capacity type. // If spot is not used, the instanceTypes are not required to be sorted diff --git a/pkg/cloudprovider/aws/instancetypes.go b/pkg/cloudprovider/aws/instancetypes.go index 599dccef0cdc..a9441edbd3f2 100644 --- a/pkg/cloudprovider/aws/instancetypes.go +++ b/pkg/cloudprovider/aws/instancetypes.go @@ -95,6 +95,11 @@ func (p *InstanceTypeProvider) Get(ctx context.Context, provider *v1alpha1.AWS) return result, nil } +func (p *InstanceTypeProvider) Flush() { + p.cache.Flush() + p.unavailableOfferings.Flush() +} + func (p *InstanceTypeProvider) createOfferings(instanceType *InstanceType, subnetZones sets.String, availableZones sets.String) []cloudprovider.Offering { offerings := []cloudprovider.Offering{} for zone := range subnetZones.Intersection(availableZones) { @@ -133,8 +138,8 @@ func (p *InstanceTypeProvider) getInstanceTypeZones(ctx context.Context) (map[st // getInstanceTypes retrieves all instance types from the ec2 DescribeInstanceTypes API using some opinionated filters func (p *InstanceTypeProvider) getInstanceTypes(ctx context.Context) (map[string]*InstanceType, error) { - if Cached, ok := p.cache.Get(InstanceTypesCacheKey); ok { - return Cached.(map[string]*InstanceType), nil + if cached, ok := p.cache.Get(InstanceTypesCacheKey); ok { + return cached.(map[string]*InstanceType), nil } instanceTypes := map[string]*InstanceType{} if err := p.ec2api.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{ diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index 17f82998c8ff..755d6610db23 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "strings" "sync" "time" @@ -28,35 +29,39 @@ import ( "github.com/patrickmn/go-cache" "go.uber.org/zap" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes" "knative.dev/pkg/logging" "knative.dev/pkg/ptr" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" - "github.com/aws/karpenter/pkg/cloudprovider/aws/launchtemplate" + "github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver" "github.com/aws/karpenter/pkg/utils/functional" "github.com/aws/karpenter/pkg/utils/injection" ) const ( - launchTemplateNameFormat = "Karpenter-%s-%s" + launchTemplateNameFormat = "Karpenter-%s-%s" + kubernetesVersionCacheKey = "kubernetesVersion" ) type LaunchTemplateProvider struct { sync.Mutex ec2api ec2iface.EC2API - amiProvider *AMIProvider + clientSet *kubernetes.Clientset + ltResolver *ltresolver.Resolver securityGroupProvider *SecurityGroupProvider cache *cache.Cache logger *zap.SugaredLogger caBundle *string } -func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, amiProvider *AMIProvider, securityGroupProvider *SecurityGroupProvider, caBundle *string) *LaunchTemplateProvider { +func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, clientSet *kubernetes.Clientset, ltResolver *ltresolver.Resolver, securityGroupProvider *SecurityGroupProvider, caBundle *string) *LaunchTemplateProvider { l := &LaunchTemplateProvider{ ec2api: ec2api, + clientSet: clientSet, logger: logging.FromContext(ctx).Named("launchtemplate"), - amiProvider: amiProvider, + ltResolver: ltResolver, securityGroupProvider: securityGroupProvider, cache: cache.New(CacheTTL, CacheCleanupInterval), caBundle: caBundle, @@ -66,7 +71,7 @@ func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, amiP return l } -func launchTemplateName(options *launchtemplate.Resolved) string { +func launchTemplateName(options *ltresolver.ResolvedTemplate) string { hash, err := hashstructure.Hash(options, hashstructure.FormatV2, nil) if err != nil { panic(fmt.Sprintf("hashing launch template, %s", err)) @@ -88,35 +93,30 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. if err != nil { return nil, err } - // Get constrained AMI ID - amis, err := p.amiProvider.Get(ctx, constraints, instanceTypes, launchtemplate.GetAMIFamily(constraints.AMIFamily, nil)) - if err != nil { - return nil, err - } - // Construct launch templates + resolvedLaunchTemplates := p.ltResolver.Resolve(constraints, instanceTypes, <resolver.Options{ + ClusterName: injection.GetOptions(ctx).ClusterName, + ClusterEndpoint: injection.GetOptions(ctx).ClusterEndpoint, + AWSENILimitedPodDensity: injection.GetOptions(ctx).AWSENILimitedPodDensity, + InstanceProfile: instanceProfile, + SecurityGroupsIDs: securityGroupsIDs, + Tags: constraints.Tags, + Labels: functional.UnionStringMaps(constraints.Labels, additionalLabels), + CABundle: p.caBundle, + }) + launchTemplates := map[string][]cloudprovider.InstanceType{} - for amiID, instanceTypes := range amis { + for _, resolvedLaunchTemplate := range resolvedLaunchTemplates { // Ensure the launch template exists, or create it - launchTemplate, err := p.ensureLaunchTemplate(ctx, launchtemplate.Get(constraints, instanceTypes, &launchtemplate.Options{ - ClusterName: injection.GetOptions(ctx).ClusterName, - ClusterEndpoint: injection.GetOptions(ctx).ClusterEndpoint, - AWSENILimitedPodDensity: injection.GetOptions(ctx).AWSENILimitedPodDensity, - InstanceProfile: instanceProfile, - AMIID: amiID, - SecurityGroupsIDs: securityGroupsIDs, - Tags: constraints.Tags, - Labels: functional.UnionStringMaps(constraints.Labels, additionalLabels), - CABundle: p.caBundle, - })) + ec2LaunchTemplate, err := p.ensureLaunchTemplate(ctx, resolvedLaunchTemplate) if err != nil { return nil, err } - launchTemplates[aws.StringValue(launchTemplate.LaunchTemplateName)] = instanceTypes + launchTemplates[*ec2LaunchTemplate.LaunchTemplateName] = resolvedLaunchTemplate.InstanceTypes } return launchTemplates, nil } -func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, options *launchtemplate.Resolved) (*ec2.LaunchTemplate, error) { +func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, options *ltresolver.ResolvedTemplate) (*ec2.LaunchTemplate, error) { // Ensure that multiple threads don't attempt to create the same launch template p.Lock() defer p.Unlock() @@ -150,7 +150,7 @@ func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, optio return launchTemplate, nil } -func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, options *launchtemplate.Resolved) (*ec2.LaunchTemplate, error) { +func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, options *ltresolver.ResolvedTemplate) (*ec2.LaunchTemplate, error) { output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{ LaunchTemplateName: aws.String(launchTemplateName(options)), LaunchTemplateData: &ec2.RequestLaunchTemplateData{ @@ -241,3 +241,17 @@ func (p *LaunchTemplateProvider) getInstanceProfile(ctx context.Context, constra } return defaultProfile, nil } + +func (p *LaunchTemplateProvider) kubeServerVersion(ctx context.Context) (string, error) { + if version, ok := p.cache.Get(kubernetesVersionCacheKey); ok { + return version.(string), nil + } + serverVersion, err := p.clientSet.Discovery().ServerVersion() + if err != nil { + return "", err + } + version := fmt.Sprintf("%s.%s", serverVersion.Major, strings.TrimSuffix(serverVersion.Minor, "+")) + p.cache.SetDefault(kubernetesVersionCacheKey, version) + logging.FromContext(ctx).Debugf("Discovered kubernetes version %s", version) + return version, nil +} diff --git a/pkg/cloudprovider/aws/launchtemplate/al2.go b/pkg/cloudprovider/aws/ltresolver/al2.go similarity index 96% rename from pkg/cloudprovider/aws/launchtemplate/al2.go rename to pkg/cloudprovider/aws/ltresolver/al2.go index e9d0bde92280..9245fc4044e1 100644 --- a/pkg/cloudprovider/aws/launchtemplate/al2.go +++ b/pkg/cloudprovider/aws/ltresolver/al2.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package launchtemplate +package ltresolver import ( "fmt" @@ -23,7 +23,7 @@ import ( "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" - "github.com/aws/karpenter/pkg/cloudprovider/aws/launchtemplate/bootstrap" + "github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver/bootstrap" ) type AL2 struct { diff --git a/pkg/cloudprovider/aws/ltresolver/ami.go b/pkg/cloudprovider/aws/ltresolver/ami.go new file mode 100644 index 000000000000..689852f964f5 --- /dev/null +++ b/pkg/cloudprovider/aws/ltresolver/ami.go @@ -0,0 +1,52 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ltresolver + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ssm" + "github.com/aws/aws-sdk-go/service/ssm/ssmiface" + "github.com/patrickmn/go-cache" + "knative.dev/pkg/logging" + + "github.com/aws/karpenter/pkg/cloudprovider" +) + +type AMIProvider struct { + cache *cache.Cache + ssm ssmiface.SSMAPI +} + +// Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc +func (p *AMIProvider) Get(ctx context.Context, instanceType cloudprovider.InstanceType, ssmQuery string) (string, error) { + if id, ok := p.cache.Get(ssmQuery); ok { + return id.(string), nil + } + output, err := p.ssm.GetParameterWithContext(ctx, &ssm.GetParameterInput{Name: aws.String(ssmQuery)}) + if err != nil { + return "", fmt.Errorf("getting ssm parameter, %w", err) + } + ami := aws.StringValue(output.Parameter.Value) + p.cache.SetDefault(ssmQuery, ami) + logging.FromContext(ctx).Debugf("Discovered %s for query %s", ami, ssmQuery) + return ami, nil +} + +func (p *AMIProvider) Flush() { + p.cache.Flush() +} diff --git a/pkg/cloudprovider/aws/launchtemplate/bootstrap/bootstrap.go b/pkg/cloudprovider/aws/ltresolver/bootstrap/bootstrap.go similarity index 100% rename from pkg/cloudprovider/aws/launchtemplate/bootstrap/bootstrap.go rename to pkg/cloudprovider/aws/ltresolver/bootstrap/bootstrap.go diff --git a/pkg/cloudprovider/aws/launchtemplate/bootstrap/bottlerocket.go b/pkg/cloudprovider/aws/ltresolver/bootstrap/bottlerocket.go similarity index 100% rename from pkg/cloudprovider/aws/launchtemplate/bootstrap/bottlerocket.go rename to pkg/cloudprovider/aws/ltresolver/bootstrap/bottlerocket.go diff --git a/pkg/cloudprovider/aws/launchtemplate/bootstrap/eksbootstrap.go b/pkg/cloudprovider/aws/ltresolver/bootstrap/eksbootstrap.go similarity index 100% rename from pkg/cloudprovider/aws/launchtemplate/bootstrap/eksbootstrap.go rename to pkg/cloudprovider/aws/ltresolver/bootstrap/eksbootstrap.go diff --git a/pkg/cloudprovider/aws/launchtemplate/bottlerocket.go b/pkg/cloudprovider/aws/ltresolver/bottlerocket.go similarity index 96% rename from pkg/cloudprovider/aws/launchtemplate/bottlerocket.go rename to pkg/cloudprovider/aws/ltresolver/bottlerocket.go index 5d7b882f299c..97938d5855f7 100644 --- a/pkg/cloudprovider/aws/launchtemplate/bottlerocket.go +++ b/pkg/cloudprovider/aws/ltresolver/bottlerocket.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package launchtemplate +package ltresolver import ( "fmt" @@ -24,7 +24,7 @@ import ( "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" - "github.com/aws/karpenter/pkg/cloudprovider/aws/launchtemplate/bootstrap" + "github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver/bootstrap" ) type Bottlerocket struct { diff --git a/pkg/cloudprovider/aws/launchtemplate/launchtemplate.go b/pkg/cloudprovider/aws/ltresolver/resolver.go similarity index 50% rename from pkg/cloudprovider/aws/launchtemplate/launchtemplate.go rename to pkg/cloudprovider/aws/ltresolver/resolver.go index c26f24ec3569..bf77b5808da2 100644 --- a/pkg/cloudprovider/aws/launchtemplate/launchtemplate.go +++ b/pkg/cloudprovider/aws/ltresolver/resolver.go @@ -12,18 +12,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package launchtemplate +package ltresolver import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ssm/ssmiface" + "github.com/patrickmn/go-cache" core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" - "github.com/aws/karpenter/pkg/cloudprovider/aws/launchtemplate/bootstrap" + "github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver/bootstrap" ) var defaultEBS = v1alpha1.BlockDevice{ @@ -32,51 +34,85 @@ var defaultEBS = v1alpha1.BlockDevice{ VolumeSize: resource.NewScaledQuantity(20, resource.Giga), } +// Resolver is able to fill-in dynamic launch template parameters +type Resolver struct { + amiProvider *AMIProvider +} + +// Options define the static launch template parameters type Options struct { ClusterName string ClusterEndpoint string AWSENILimitedPodDensity bool InstanceProfile string + KubernetesVersion string // Level-triggered fields that may change out of sync. - AMIID string SecurityGroupsIDs []string Tags map[string]string Labels map[string]string `hash:"ignore"` CABundle *string `hash:"ignore"` } -type Resolved struct { +// ResolvedTemplate holds the dynamically generated launch tempalte parameters +type ResolvedTemplate struct { *Options UserData bootstrap.Bootstrapper BlockDeviceMappings []*v1alpha1.BlockDeviceMapping MetadataOptions *v1alpha1.MetadataOptions + AMIID string + InstanceTypes []cloudprovider.InstanceType `hash:"ignore"` } +// AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters type AMIFamily interface { UserData(kubeletConfig v1alpha5.KubeletConfiguration, taints []core.Taint, labels map[string]string, caBundle *string) bootstrap.Bootstrapper - SSMAlias(version string, instanceTypes cloudprovider.InstanceType) string + SSMAlias(version string, instanceType cloudprovider.InstanceType) string DefaultBlockDeviceMappings() []*v1alpha1.BlockDeviceMapping DefaultMetadataOptions() *v1alpha1.MetadataOptions } -func Get(constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, options *Options) *Resolved { - amiFamily := GetAMIFamily(constraints.AMIFamily, options) - resolved := &Resolved{ - Options: options, - UserData: amiFamily.UserData(constraints.KubeletConfiguration, constraints.Taints, options.Labels, options.CABundle), - BlockDeviceMappings: constraints.BlockDeviceMappings, - MetadataOptions: constraints.MetadataOptions, +// New constructs a new launch template Resolver +func New(ssm ssmiface.SSMAPI, c *cache.Cache) *Resolver { + return &Resolver{ + amiProvider: &AMIProvider{ + ssm: ssm, + cache: c, + }, } - if resolved.BlockDeviceMappings == nil { - resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings() +} + +// Resolve generates launch templates using the static options and dynamically generates launch template parameters. +// Multiple ResolvedTemplates are returned based on the instanceTypes passed in to support special AMIs for certain instance types like GPUs. +func (r Resolver) Resolve(constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, options *Options) []*ResolvedTemplate { + amiFamily := r.getAMIFamily(constraints.AMIFamily, options) + amiIDs := map[string][]cloudprovider.InstanceType{} + for _, instanceType := range instanceTypes { + amiID := amiFamily.SSMAlias(options.KubernetesVersion, instanceType) + amiIDs[amiID] = append(amiIDs[amiID], instanceType) } - if resolved.MetadataOptions == nil { - resolved.MetadataOptions = amiFamily.DefaultMetadataOptions() + var resolvedTemplates []*ResolvedTemplate + for amiID, instanceTypes := range amiIDs { + resolved := &ResolvedTemplate{ + Options: options, + UserData: amiFamily.UserData(constraints.KubeletConfiguration, constraints.Taints, options.Labels, options.CABundle), + BlockDeviceMappings: constraints.BlockDeviceMappings, + MetadataOptions: constraints.MetadataOptions, + AMIID: amiID, + InstanceTypes: instanceTypes, + } + if resolved.BlockDeviceMappings == nil { + resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings() + } + if resolved.MetadataOptions == nil { + resolved.MetadataOptions = amiFamily.DefaultMetadataOptions() + } + resolvedTemplates = append(resolvedTemplates, resolved) } - return resolved + + return resolvedTemplates } -func GetAMIFamily(amiFamily *string, options *Options) AMIFamily { +func (r Resolver) getAMIFamily(amiFamily *string, options *Options) AMIFamily { switch aws.StringValue(amiFamily) { case v1alpha1.AMIFamilyAL2: return &AL2{Options: options} diff --git a/pkg/cloudprovider/aws/launchtemplate/ubuntu.go b/pkg/cloudprovider/aws/ltresolver/ubuntu.go similarity index 95% rename from pkg/cloudprovider/aws/launchtemplate/ubuntu.go rename to pkg/cloudprovider/aws/ltresolver/ubuntu.go index 9a0ccaaad400..2298a7ae7343 100644 --- a/pkg/cloudprovider/aws/launchtemplate/ubuntu.go +++ b/pkg/cloudprovider/aws/ltresolver/ubuntu.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package launchtemplate +package ltresolver import ( "fmt" @@ -23,7 +23,7 @@ import ( "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" - "github.com/aws/karpenter/pkg/cloudprovider/aws/launchtemplate/bootstrap" + "github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver/bootstrap" ) type Ubuntu struct { diff --git a/pkg/cloudprovider/aws/securitygroups.go b/pkg/cloudprovider/aws/securitygroups.go index ab892d757bf8..bb8fa057c925 100644 --- a/pkg/cloudprovider/aws/securitygroups.go +++ b/pkg/cloudprovider/aws/securitygroups.go @@ -58,6 +58,10 @@ func (s *SecurityGroupProvider) Get(ctx context.Context, constraints *v1alpha1.C return securityGroupIds, nil } +func (p *SecurityGroupProvider) Flush() { + p.cache.Flush() +} + func (s *SecurityGroupProvider) getFilters(constraints *v1alpha1.Constraints) []*ec2.Filter { filters := []*ec2.Filter{} for key, value := range constraints.SecurityGroupSelector { diff --git a/pkg/cloudprovider/aws/subnets.go b/pkg/cloudprovider/aws/subnets.go index 2a0b2877de8e..8382ee9b4ffb 100644 --- a/pkg/cloudprovider/aws/subnets.go +++ b/pkg/cloudprovider/aws/subnets.go @@ -41,27 +41,31 @@ func NewSubnetProvider(ec2api ec2iface.EC2API) *SubnetProvider { } } -func (s *SubnetProvider) Get(ctx context.Context, constraints *v1alpha1.AWS) ([]*ec2.Subnet, error) { +func (p *SubnetProvider) Get(ctx context.Context, constraints *v1alpha1.AWS) ([]*ec2.Subnet, error) { filters := getFilters(constraints) hash, err := hashstructure.Hash(filters, hashstructure.FormatV2, nil) if err != nil { return nil, err } - if subnets, ok := s.cache.Get(fmt.Sprint(hash)); ok { + if subnets, ok := p.cache.Get(fmt.Sprint(hash)); ok { return subnets.([]*ec2.Subnet), nil } - output, err := s.ec2api.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{Filters: filters}) + output, err := p.ec2api.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{Filters: filters}) if err != nil { return nil, fmt.Errorf("describing subnets %s, %w", pretty.Concise(filters), err) } if len(output.Subnets) == 0 { return nil, fmt.Errorf("no subnets matched selector %v", constraints.SubnetSelector) } - s.cache.SetDefault(fmt.Sprint(hash), output.Subnets) + p.cache.SetDefault(fmt.Sprint(hash), output.Subnets) logging.FromContext(ctx).Debugf("Discovered subnets: %s", prettySubnets(output.Subnets)) return output.Subnets, nil } +func (p *SubnetProvider) Flush() { + p.cache.Flush() +} + func getFilters(constraints *v1alpha1.AWS) []*ec2.Filter { filters := []*ec2.Filter{} // Filter by subnet diff --git a/pkg/cloudprovider/aws/suite_test.go b/pkg/cloudprovider/aws/suite_test.go index d94c07b2218e..d988dee76a8f 100644 --- a/pkg/cloudprovider/aws/suite_test.go +++ b/pkg/cloudprovider/aws/suite_test.go @@ -28,6 +28,7 @@ import ( "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" "github.com/aws/karpenter/pkg/cloudprovider/aws/fake" + "github.com/aws/karpenter/pkg/cloudprovider/aws/ltresolver" "github.com/aws/karpenter/pkg/cloudprovider/registry" "github.com/aws/karpenter/pkg/controllers/provisioning" "github.com/aws/karpenter/pkg/controllers/selection" @@ -58,6 +59,7 @@ var env *test.Environment var launchTemplateCache *cache.Cache var securityGroupCache *cache.Cache var subnetCache *cache.Cache +var amiCache *cache.Cache var unavailableOfferingsCache *cache.Cache var fakeEC2API *fake.EC2API var provisioners *provisioning.Controller @@ -84,6 +86,7 @@ var _ = BeforeSuite(func() { unavailableOfferingsCache = cache.New(InsufficientCapacityErrorCacheTTL, InsufficientCapacityErrorCacheCleanupInterval) securityGroupCache = cache.New(CacheTTL, CacheCleanupInterval) subnetCache = cache.New(CacheTTL, CacheCleanupInterval) + amiCache = cache.New(CacheTTL, CacheCleanupInterval) fakeEC2API = &fake.EC2API{} subnetProvider := &SubnetProvider{ ec2api: fakeEC2API, @@ -106,7 +109,7 @@ var _ = BeforeSuite(func() { instanceProvider: &InstanceProvider{ fakeEC2API, instanceTypeProvider, subnetProvider, &LaunchTemplateProvider{ ec2api: fakeEC2API, - amiProvider: NewAMIProvider(&fake.SSMAPI{}, clientSet), + ltResolver: ltresolver.New(fake.SSMAPI{}, amiCache), securityGroupProvider: securityGroupProvider, cache: launchTemplateCache, caBundle: ptr.String("ca-bundle"), @@ -140,6 +143,7 @@ var _ = Describe("Allocation", func() { securityGroupCache.Flush() subnetCache.Flush() unavailableOfferingsCache.Flush() + amiCache.Flush() }) AfterEach(func() {