Skip to content

Commit

Permalink
Validation step to check Nodes and ASG launch configs (#112)
Browse files Browse the repository at this point in the history 
* Validation step to check Nodes and ASG launch configs

* Validating launch definition after a rolling upgrade

* Resolve error log message and return statement

Co-authored-by: Eytan Avisror <[email protected]>
  • Loading branch information
@shreyas-badiger @eytan-avisror
shreyas-badiger and eytan-avisror authored Aug 24, 2020
1 parent 03a402c commit 329e60f
Show file tree
Hide file tree
Showing 2 changed files with 234 additions and 23 deletions.
38 changes: 38 additions & 0 deletions controllers/rollingupgrade_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -704,9 +704,47 @@ func (r *RollingUpgradeReconciler) Process(ctx *context.Context,
return
}

//Validation step: check if all the nodes have the latest launchconfig.
r.info(ruObj, "Validating the launch definition of nodes and ASG")
if err := r.validateNodesLaunchDefinition(ruObj); err != nil {
r.error(ruObj, err, "Launch definition validation failed")
r.finishExecution(StatusError, nodesProcessed, ctx, ruObj)
return
}

r.finishExecution(StatusComplete, nodesProcessed, ctx, ruObj)
}

//Check if ec2Instances and the ASG have same launch config.
func (r *RollingUpgradeReconciler) validateNodesLaunchDefinition(ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
//Get ASG launch config
var err error
err = r.populateAsg(ruObj)
if err != nil {
return errors.New("Unable to populate the ASG object")
}
asg, err := r.GetAutoScalingGroup(ruObj.Name)
if err != nil {
return fmt.Errorf("Unable to load ASG with name: %s", ruObj.Name)
}
launchDefinition := NewLaunchDefinition(asg)
launchConfigASG, launchTemplateASG := launchDefinition.launchConfigurationName, launchDefinition.launchTemplate

//Get ec2 instances and their launch configs.
ec2instances := asg.Instances
for _, ec2Instance := range ec2instances {
ec2InstanceID, ec2InstanceLaunchConfig, ec2InstanceLaunchTemplate := ec2Instance.InstanceId, ec2Instance.LaunchConfigurationName, ec2Instance.LaunchTemplate
if aws.StringValue(launchConfigASG) != aws.StringValue(ec2InstanceLaunchConfig) {
return fmt.Errorf("launch config mismatch, %s instance config - %s, does not match the asg config", aws.StringValue(ec2InstanceID), aws.StringValue(ec2InstanceLaunchConfig))
} else if launchTemplateASG != nil && ec2InstanceLaunchTemplate != nil {
if aws.StringValue(launchTemplateASG.LaunchTemplateId) != aws.StringValue(ec2InstanceLaunchTemplate.LaunchTemplateId) {
return fmt.Errorf("launch template mismatch, %s instance template - %s, does not match the asg template", aws.StringValue(ec2InstanceID), aws.StringValue(ec2InstanceLaunchTemplate.LaunchTemplateId))
}
}
}
return nil
}

// MarkObjForCleanup sets the annotation on the given object for deletion.
func MarkObjForCleanup(ruObj *upgrademgrv1alpha1.RollingUpgrade) {
if ruObj.ObjectMeta.Annotations == nil {
Expand Down
219 changes: 196 additions & 23 deletions controllers/rollingupgrade_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,9 +319,10 @@ type MockEC2 struct {

type MockAutoscalingGroup struct {
autoscalingiface.AutoScalingAPI
errorFlag bool
awsErr awserr.Error
errorInstanceId string
errorFlag bool
awsErr awserr.Error
errorInstanceId string
autoScalingGroups []*autoscaling.Group
}

func (m MockEC2) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) {
Expand All @@ -347,25 +348,17 @@ func (mockAutoscalingGroup MockAutoscalingGroup) EnterStandby(input *autoscaling
}

func (mockAutoscalingGroup MockAutoscalingGroup) DescribeAutoScalingGroups(input *autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error) {
var err error
output := autoscaling.DescribeAutoScalingGroupsOutput{
AutoScalingGroups: []*autoscaling.Group{},
}
//To support parallel ASG tracking.
asgA, asgB := "asg-a", "asg-b"

correctAsg := "correct-asg"
tooMany := "too-many"
asgA := "asg-a"
asgB := "asg-b"

if mockAutoscalingGroup.errorFlag {
err = mockAutoscalingGroup.awsErr
}
switch *input.AutoScalingGroupNames[0] {
case correctAsg:
output.AutoScalingGroups = []*autoscaling.Group{
{AutoScalingGroupName: &correctAsg},
}
case tooMany:
output.AutoScalingGroups = []*autoscaling.Group{
{AutoScalingGroupName: &tooMany},
{AutoScalingGroupName: &tooMany},
}
case asgA:
output.AutoScalingGroups = []*autoscaling.Group{
{AutoScalingGroupName: &asgA},
Expand All @@ -375,10 +368,9 @@ func (mockAutoscalingGroup MockAutoscalingGroup) DescribeAutoScalingGroups(input
{AutoScalingGroupName: &asgB},
}
default:
output.AutoScalingGroups = []*autoscaling.Group{}
output.AutoScalingGroups = mockAutoscalingGroup.autoScalingGroups
}

return &output, nil
return &output, err
}

func (mockAutoscalingGroup MockAutoscalingGroup) TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) {
Expand Down Expand Up @@ -710,21 +702,28 @@ func TestGetNodeFromAsgMissingNode(t *testing.T) {
func TestPopulateAsgSuccess(t *testing.T) {
g := gomega.NewGomegaWithT(t)

correctAsg := "correct-asg"
mockAsg := &autoscaling.Group{
AutoScalingGroupName: &correctAsg,
}
mockAsgClient := MockAutoscalingGroup{
autoScalingGroups: []*autoscaling.Group{mockAsg},
}

ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
TypeMeta: metav1.TypeMeta{Kind: "RollingUpgrade", APIVersion: "v1alpha1"},
Spec: upgrademgrv1alpha1.RollingUpgradeSpec{AsgName: "correct-asg"}}

rcRollingUpgrade := &RollingUpgradeReconciler{
Log: log2.NullLogger{},
ClusterState: NewClusterState(),
ASGClient: &MockAutoscalingGroup{},
ASGClient: mockAsgClient,
EC2Client: MockEC2{},
}
err := rcRollingUpgrade.populateAsg(ruObj)

g.Expect(err).To(gomega.BeNil())

correctAsg := "correct-asg"
expectedAsg := autoscaling.Group{AutoScalingGroupName: &correctAsg}

requestedAsg, ok := rcRollingUpgrade.ruObjNameToASG.Load(ruObj.Name)
Expand All @@ -735,14 +734,23 @@ func TestPopulateAsgSuccess(t *testing.T) {
func TestPopulateAsgTooMany(t *testing.T) {
g := gomega.NewGomegaWithT(t)

mockAsg1 := &autoscaling.Group{
AutoScalingGroupName: aws.String("too-many"),
}
mockAsg2 := &autoscaling.Group{
AutoScalingGroupName: aws.String("too-many"),
}
mockAsgClient := MockAutoscalingGroup{
autoScalingGroups: []*autoscaling.Group{mockAsg1, mockAsg2},
}
ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
TypeMeta: metav1.TypeMeta{Kind: "RollingUpgrade", APIVersion: "v1alpha1"},
Spec: upgrademgrv1alpha1.RollingUpgradeSpec{AsgName: "too-many"}}

rcRollingUpgrade := &RollingUpgradeReconciler{
Log: log2.NullLogger{},
ClusterState: NewClusterState(),
ASGClient: &MockAutoscalingGroup{},
ASGClient: mockAsgClient,
EC2Client: MockEC2{},
}
err := rcRollingUpgrade.populateAsg(ruObj)
Expand Down Expand Up @@ -2693,3 +2701,168 @@ func TestUpdateInstancesNotExists(t *testing.T) {

g.Expect(processCount).To(gomega.Equal(1))
}

func TestValidateNodesLaunchDefinitionSameLaunchConfig(t *testing.T) {
g := gomega.NewGomegaWithT(t)

someLaunchConfig := "some-launch-config"
az := "az-1"
mockInstance := autoscaling.Instance{InstanceId: aws.String("some-id"), LaunchConfigurationName: &someLaunchConfig, AvailabilityZone: &az}
mockAsg := &autoscaling.Group{
AutoScalingGroupName: aws.String("my-asg"),
LaunchConfigurationName: &someLaunchConfig,
Instances: []*autoscaling.Instance{&mockInstance},
}

ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
Spec: upgrademgrv1alpha1.RollingUpgradeSpec{AsgName: "my-asg"}}

mgr, err := buildManager()
g.Expect(err).NotTo(gomega.HaveOccurred())
c = mgr.GetClient()

mockAsgClient := MockAutoscalingGroup{
autoScalingGroups: []*autoscaling.Group{mockAsg},
}

rcRollingUpgrade := &RollingUpgradeReconciler{
Client: mgr.GetClient(),
Log: log2.NullLogger{},
ASGClient: mockAsgClient,
EC2Client: MockEC2{},
generatedClient: kubernetes.NewForConfigOrDie(mgr.GetConfig()),
admissionMap: sync.Map{},
ruObjNameToASG: sync.Map{},
ClusterState: NewClusterState(),
CacheConfig: cache.NewConfig(0*time.Second, 0, 0),
}
rcRollingUpgrade.admissionMap.Store(ruObj.Name, "processing")
rcRollingUpgrade.ruObjNameToASG.Store(ruObj.Name, &mockAsg)

// This execution should not perform drain or termination, but should pass
err = rcRollingUpgrade.validateNodesLaunchDefinition(ruObj)
g.Expect(err).To(gomega.BeNil())
}

func TestValidateNodesLaunchDefinitionDifferentLaunchConfig(t *testing.T) {
g := gomega.NewGomegaWithT(t)

someLaunchConfig := "some-launch-config"
someOtherLaunchConfig := "some-other-launch-config"
az := "az-1"
mockInstance := autoscaling.Instance{InstanceId: aws.String("some-id"), LaunchConfigurationName: &someLaunchConfig, AvailabilityZone: &az}
mockAsg := &autoscaling.Group{
AutoScalingGroupName: aws.String("my-asg"),
LaunchConfigurationName: &someOtherLaunchConfig,
Instances: []*autoscaling.Instance{&mockInstance}}

ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
Spec: upgrademgrv1alpha1.RollingUpgradeSpec{AsgName: "my-asg"}}

mgr, err := buildManager()
g.Expect(err).NotTo(gomega.HaveOccurred())
c = mgr.GetClient()

mockAsgClient := MockAutoscalingGroup{
autoScalingGroups: []*autoscaling.Group{mockAsg},
}

rcRollingUpgrade := &RollingUpgradeReconciler{
Client: mgr.GetClient(),
Log: log2.NullLogger{},
ASGClient: mockAsgClient,
EC2Client: MockEC2{},
generatedClient: kubernetes.NewForConfigOrDie(mgr.GetConfig()),
admissionMap: sync.Map{},
ruObjNameToASG: sync.Map{},
ClusterState: NewClusterState(),
CacheConfig: cache.NewConfig(0*time.Second, 0, 0),
}
rcRollingUpgrade.admissionMap.Store(ruObj.Name, "processing")
rcRollingUpgrade.ruObjNameToASG.Store(ruObj.Name, &mockAsg)

// This execution should not perform drain or termination, but should pass
err = rcRollingUpgrade.validateNodesLaunchDefinition(ruObj)
g.Expect(err).To(gomega.Not(gomega.BeNil()))
}

func TestValidateNodesLaunchDefinitionSameLaunchTemplate(t *testing.T) {
g := gomega.NewGomegaWithT(t)
someLaunchTemplate := &autoscaling.LaunchTemplateSpecification{LaunchTemplateId: aws.String("launch-template-id-v1")}
az := "az-1"
mockInstance := autoscaling.Instance{InstanceId: aws.String("some-id"), LaunchTemplate: someLaunchTemplate, AvailabilityZone: &az}
mockAsg := &autoscaling.Group{
AutoScalingGroupName: aws.String("my-asg"),
LaunchTemplate: someLaunchTemplate,
Instances: []*autoscaling.Instance{&mockInstance}}

ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
Spec: upgrademgrv1alpha1.RollingUpgradeSpec{AsgName: "my-asg"}}

mgr, err := buildManager()
g.Expect(err).NotTo(gomega.HaveOccurred())
c = mgr.GetClient()

mockAsgClient := MockAutoscalingGroup{
autoScalingGroups: []*autoscaling.Group{mockAsg},
}

rcRollingUpgrade := &RollingUpgradeReconciler{
Client: mgr.GetClient(),
Log: log2.NullLogger{},
ASGClient: mockAsgClient,
EC2Client: MockEC2{},
generatedClient: kubernetes.NewForConfigOrDie(mgr.GetConfig()),
admissionMap: sync.Map{},
ruObjNameToASG: sync.Map{},
ClusterState: NewClusterState(),
CacheConfig: cache.NewConfig(0*time.Second, 0, 0),
}
rcRollingUpgrade.admissionMap.Store(ruObj.Name, "processing")
rcRollingUpgrade.ruObjNameToASG.Store(ruObj.Name, &mockAsg)

// This execution should not perform drain or termination, but should pass
err = rcRollingUpgrade.validateNodesLaunchDefinition(ruObj)
g.Expect(err).To(gomega.BeNil())
}

func TestValidateNodesLaunchDefinitionDifferentLaunchTemplate(t *testing.T) {
g := gomega.NewGomegaWithT(t)
someLaunchTemplate := &autoscaling.LaunchTemplateSpecification{LaunchTemplateId: aws.String("launch-template-id-v1")}
someOtherLaunchTemplate := &autoscaling.LaunchTemplateSpecification{LaunchTemplateId: aws.String("launch-template-id-v2")}
az := "az-1"
mockInstance := autoscaling.Instance{InstanceId: aws.String("some-id"), LaunchTemplate: someLaunchTemplate, AvailabilityZone: &az}
mockAsg := &autoscaling.Group{
AutoScalingGroupName: aws.String("my-asg"),
LaunchTemplate: someOtherLaunchTemplate,
Instances: []*autoscaling.Instance{&mockInstance}}

ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
Spec: upgrademgrv1alpha1.RollingUpgradeSpec{AsgName: "my-asg"}}

mgr, err := buildManager()
g.Expect(err).NotTo(gomega.HaveOccurred())
c = mgr.GetClient()

mockAsgClient := MockAutoscalingGroup{
autoScalingGroups: []*autoscaling.Group{mockAsg},
}

rcRollingUpgrade := &RollingUpgradeReconciler{
Client: mgr.GetClient(),
Log: log2.NullLogger{},
ASGClient: mockAsgClient,
EC2Client: MockEC2{},
generatedClient: kubernetes.NewForConfigOrDie(mgr.GetConfig()),
admissionMap: sync.Map{},
ruObjNameToASG: sync.Map{},
ClusterState: NewClusterState(),
CacheConfig: cache.NewConfig(0*time.Second, 0, 0),
}
rcRollingUpgrade.admissionMap.Store(ruObj.Name, "processing")
rcRollingUpgrade.ruObjNameToASG.Store(ruObj.Name, &mockAsg)

// This execution should not perform drain or termination, but should pass
err = rcRollingUpgrade.validateNodesLaunchDefinition(ruObj)
g.Expect(err).To(gomega.Not(gomega.BeNil()))
}

0 comments on commit 329e60f

Please sign in to comment.