Skip to content

Commit

Permalink
Migrate Script Runner (#179)
Browse files Browse the repository at this point in the history
* Basic script runner

Signed-off-by: Eytan Avisror <[email protected]>

* Update upgrade.go

Signed-off-by: Eytan Avisror <[email protected]>
  • Loading branch information
eytan-avisror authored Feb 10, 2021
1 parent 2c1d8e7 commit dd6a332
Show file tree
Hide file tree
Showing 11 changed files with 279 additions and 7 deletions.
32 changes: 32 additions & 0 deletions api/v1alpha1/rollingupgrade_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,17 @@ type RollingUpgradeStatus struct {
LastNodeDrainTime metav1.Time `json:"lastDrainTime,omitempty"`
}

func (s *RollingUpgradeStatus) SetCondition(cond RollingUpgradeCondition) {
// if condition exists, overwrite, otherwise append
for ix, c := range s.Conditions {
if c.Type == cond.Type {
s.Conditions[ix] = cond
return
}
}
s.Conditions = append(s.Conditions, cond)
}

// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:path=rollingupgrades,scope=Namespaced,shortName=ru
Expand Down Expand Up @@ -160,6 +171,22 @@ func (r *RollingUpgrade) ScalingGroupName() string {
return r.Spec.AsgName
}

func (r *RollingUpgrade) PostTerminateScript() string {
return r.Spec.PostTerminate.Script
}

func (r *RollingUpgrade) PostWaitScript() string {
return r.Spec.PostDrain.PostWaitScript
}

func (r *RollingUpgrade) PreDrainScript() string {
return r.Spec.PreDrain.Script
}

func (r *RollingUpgrade) PostDrainScript() string {
return r.Spec.PostDrain.Script
}

func (r *RollingUpgrade) CurrentStatus() string {
return r.Status.CurrentStatus
}
Expand Down Expand Up @@ -216,13 +243,18 @@ func (r *RollingUpgrade) SetNodesProcessed(n int) {
r.Status.NodesProcessed = n
}

func (r *RollingUpgrade) GetStatus() RollingUpgradeStatus {
return r.Status
}

func (r *RollingUpgrade) IsForceRefresh() bool {
return r.Spec.ForceRefresh
}

func (r *RollingUpgrade) StrategyMode() UpdateStrategyMode {
return r.Spec.Strategy.Mode
}

func (r *RollingUpgrade) Validate() (bool, error) {
strategy := r.Spec.Strategy

Expand Down
2 changes: 2 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions config/crd/bases/upgrademgr.keikoproj.io_rollingupgrades.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,12 @@ spec:
type: string
endTime:
type: string
lastDrainTime:
format: date-time
type: string
lastTerminationTime:
format: date-time
type: string
nodesProcessed:
type: integer
startTime:
Expand Down
4 changes: 4 additions & 0 deletions controllers/providers/aws/ec2.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ func (a *AmazonClientSet) DescribeTaggedInstanceIDs(tagKey, tagValue string) ([]
Name: aws.String(key),
Values: aws.StringSlice([]string{tagValue}),
},
{
Name: aws.String("instance-state-name"),
Values: aws.StringSlice([]string{"pending", "running"}),
},
},
}

Expand Down
10 changes: 10 additions & 0 deletions controllers/providers/aws/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,16 @@ func GetScalingAZs(instances []*autoscaling.Instance) []string {
return AZs
}

func GetInstanceIDs(instances []*autoscaling.Instance) []string {
IDs := make([]string, 0)
for _, instance := range instances {
ID := aws.StringValue(instance.InstanceId)
IDs = append(IDs, ID)
}
sort.Strings(IDs)
return IDs
}

// func SelectInstancesByAZ(instances []*autoscaling.Group) *autoscaling.Instance {
// for _, instance := range group.Instances {
// selectedID := aws.StringValue(instance.InstanceId)
Expand Down
6 changes: 3 additions & 3 deletions controllers/rollingupgrade_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"github.com/go-logr/logr"
"github.com/keikoproj/aws-sdk-go-cache/cache"
"github.com/keikoproj/upgrade-manager/api/v1alpha1"
upgrademgrv1alpha1 "github.com/keikoproj/upgrade-manager/api/v1alpha1"
"github.com/keikoproj/upgrade-manager/controllers/common"
awsprovider "github.com/keikoproj/upgrade-manager/controllers/providers/aws"
kubeprovider "github.com/keikoproj/upgrade-manager/controllers/providers/kubernetes"
Expand All @@ -48,6 +47,7 @@ type RollingUpgradeReconciler struct {
Cloud *DiscoveredState
EventWriter *kubeprovider.EventWriter
maxParallel int
ScriptRunner ScriptRunner
}

type RollingUpgradeAuthenticator struct {
Expand All @@ -67,7 +67,7 @@ type RollingUpgradeAuthenticator struct {
// Reconcile reads that state of the cluster for a RollingUpgrade object and makes changes based on the state read
// and the details in the RollingUpgrade.Spec
func (r *RollingUpgradeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
rollingUpgrade := &upgrademgrv1alpha1.RollingUpgrade{}
rollingUpgrade := &v1alpha1.RollingUpgrade{}
err := r.Get(ctx, req.NamespacedName, rollingUpgrade)
if err != nil {
if kerrors.IsNotFound(err) {
Expand Down Expand Up @@ -144,7 +144,7 @@ func (r *RollingUpgradeReconciler) Reconcile(ctx context.Context, req ctrl.Reque
// SetupWithManager sets up the controller with the Manager.
func (r *RollingUpgradeReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&upgrademgrv1alpha1.RollingUpgrade{}).
For(&v1alpha1.RollingUpgrade{}).
WithOptions(controller.Options{MaxConcurrentReconciles: r.maxParallel}).
Complete(r)
}
Expand Down
133 changes: 133 additions & 0 deletions controllers/script_runner.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
/*
Copyright 2019 Intuit, Inc..
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
"fmt"
"os"
"os/exec"

"github.com/go-logr/logr"
"github.com/keikoproj/upgrade-manager/api/v1alpha1"
)

const (
ShellBinary = "/bin/sh"
)

type ScriptRunner struct {
logr.Logger
}

type ScriptTarget struct {
InstanceID string
NodeName string
UpgradeObject *v1alpha1.RollingUpgrade
}

func NewScriptRunner(logger logr.Logger) ScriptRunner {
return ScriptRunner{
Logger: logger,
}
}

func (r *ScriptRunner) getEnv(target ScriptTarget) []string {
var (
asgNameEnv = "ASG_NAME"
instanceIdEnv = "INSTANCE_ID"
instanceNameEnv = "INSTANCE_NAME"
)
return []string{
fmt.Sprintf("%s=%s", asgNameEnv, target.UpgradeObject.ScalingGroupName()),
fmt.Sprintf("%s=%s", instanceIdEnv, target.InstanceID),
fmt.Sprintf("%s=%s", instanceNameEnv, target.NodeName),
}
}

func (r *ScriptRunner) runScript(script string, target ScriptTarget) (string, error) {
r.Info("running script", "script", script, "name", target.UpgradeObject.NamespacedName())
command := exec.Command(ShellBinary, "-c", script)
command.Env = append(os.Environ(), r.getEnv(target)...)

out, err := command.CombinedOutput()
if err != nil {
return string(out), err
}
return string(out), nil
}

func (r *ScriptRunner) PostTerminate(target ScriptTarget) error {
script := target.UpgradeObject.PostTerminateScript()
if script == "" {
return nil
}

out, err := r.runScript(script, target)
if err != nil {
r.Info("script execution failed", "output", out, "stage", "PostTerminate", "script", script, "name", target.UpgradeObject.NamespacedName(), "target", target.NodeName)
return err
}

r.Info("script execution succeeded", "output", out, "stage", "PostTerminate", "script", script, "name", target.UpgradeObject.NamespacedName(), "target", target.NodeName)

return nil
}

func (r *ScriptRunner) PreDrain(target ScriptTarget) error {
script := target.UpgradeObject.PreDrainScript()
if script == "" {
return nil
}

out, err := r.runScript(script, target)
if err != nil {
r.Info("script execution failed", "output", out, "stage", "PreDrain", "script", script, "name", target.UpgradeObject.NamespacedName(), "target", target.NodeName)
return err
}

r.Info("script execution succeeded", "output", out, "stage", "PreDrain", "script", script, "name", target.UpgradeObject.NamespacedName(), "target", target.NodeName)
return nil
}

func (r *ScriptRunner) PostDrain(target ScriptTarget) error {
script := target.UpgradeObject.PostDrainScript()
if script == "" {
return nil
}

out, err := r.runScript(script, target)
if err != nil {
r.Info("script execution failed", "output", out, "stage", "PostDrain", "script", script, "name", target.UpgradeObject.NamespacedName(), "target", target.NodeName)
return err
}

r.Info("script execution succeeded", "output", out, "stage", "PostDrain", "script", script, "name", target.UpgradeObject.NamespacedName(), "target", target.NodeName)
return nil
}

func (r *ScriptRunner) PostWait(target ScriptTarget) error {
script := target.UpgradeObject.PostWaitScript()
if script == "" {
return nil
}

out, err := r.runScript(script, target)
if err != nil {
r.Info("script execution failed", "output", out, "stage", "PostWait", "script", script, "name", target.UpgradeObject.NamespacedName(), "target", target.NodeName)
return err
}

r.Info("script execution succeeded", "output", out, "stage", "PostWait", "script", script, "name", target.UpgradeObject.NamespacedName(), "target", target.NodeName)
return nil
}
48 changes: 48 additions & 0 deletions controllers/script_runner_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package controllers

import (
"testing"

"github.com/keikoproj/upgrade-manager/api/v1alpha1"
"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
runtimelog "sigs.k8s.io/controller-runtime/pkg/log"
)

func TestScriptSuccess(t *testing.T) {
g := gomega.NewGomegaWithT(t)
r := &ScriptRunner{Logger: runtimelog.NullLogger{}}
target := ScriptTarget{
InstanceID: "instance",
NodeName: "node",
UpgradeObject: &v1alpha1.RollingUpgrade{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Namespace: "default",
},
},
}

out, err := r.runScript("echo hello", target)
g.Expect(err).To(gomega.BeNil())
g.Expect(out).To(gomega.Equal("hello\n"))
}

func TestScriptFailure(t *testing.T) {
g := gomega.NewGomegaWithT(t)
r := &ScriptRunner{Logger: runtimelog.NullLogger{}}
target := ScriptTarget{
InstanceID: "instance",
NodeName: "node",
UpgradeObject: &v1alpha1.RollingUpgrade{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Namespace: "default",
},
},
}

out, err := r.runScript("echo this will fail; exit 1", target)
g.Expect(err).To(gomega.Not(gomega.BeNil()))
g.Expect(out).To(gomega.Not(gomega.Equal("")))
}
Loading

0 comments on commit dd6a332

Please sign in to comment.