Merge pull request #20 from silinternational/develop

Release 0.2.0 - better calculations for RightSizeAsgForEcsCluster

README.md

@@ -32,8 +32,10 @@ Usage:
   awsops [command]
 
 Available Commands:
+  completion  Generate the autocompletion script for the specified shell
   ecs         ECS related actions, run 'awsops ecs' to view list of subcommands
   help        Help about any command
+  lambda      Commands for interacting with Lambda service
 
 Flags:
       --config string    config file (default is $HOME/.awsops.yaml)
@@ -88,7 +90,7 @@ Global Flags:
 ```
 
 ```
-$ awsops ecs replaceInstances --help
+$ `awsops ecs replaceInstances --help`
 Gracefully replace EC2 instances for given ECS cluster
 
 Usage:
@@ -107,8 +109,8 @@ Global Flags:
 ```
 $ awsops ecs rightSizeCluster --help
 This command calculates total memory and CPU needed
-for all services in the given ECS cluster and then adjusts
-instance count in the ASG based on instance type/size to
+for all services in the given ECS cluster and then adjusts 
+instance count in the ASG based on instance type/size to 
 support running all tasks with as few servers as is needed.
 
 This function may scale a cluster up or down depending on services.
@@ -117,11 +119,30 @@ Usage:
   awsops ecs rightSizeCluster [flags]
 
 Flags:
-  -h, --help   help for rightSizeCluster
+      --atLeastServiceDesiredCount   Ensure at least as many EC2 instances as largest ECS service desired count.
+  -h, --help                         help for rightSizeCluster
 
 Global Flags:
   -c, --cluster string   ECS cluster name
       --config string    config file (default is $HOME/.awsops.yaml)
   -p, --profile string   AWS shared credentials profile to use
   -r, --region string    AWS shared credentials profile to use (default "us-east-1")
 ```
+
+```
+$ awsops lambda invoke --help
+Invoke a lambda function
+
+Usage:
+  awsops lambda invoke [flags]
+
+Flags:
+  -f, --function string   Lambda function name
+  -h, --help              help for invoke
+  -b, --payload string    Lambda function input payload as JSON string
+
+Global Flags:
+      --config string    config file (default is $HOME/.awsops.yaml)
+  -p, --profile string   AWS shared credentials profile to use
+  -r, --region string    AWS shared credentials profile to use (default "us-east-1")
+```

cmd/ecsReplaceInstances.go

@@ -16,12 +16,13 @@ package cmd
 
 import (
 	"fmt"
-	"os"
+	"log"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/silinternational/awsops/lib"
 	"github.com/spf13/cobra"
+
+	"github.com/silinternational/awsops/lib"
 )
 
 // ecsReplaceInstancesCmd represents the ecsReplaceInstances command
@@ -30,13 +31,11 @@ var replaceInstancesCmd = &cobra.Command{
 	Short: "Gracefully replace EC2 instances for given ECS cluster",
 	Long:  ``,
 	Run: func(cmd *cobra.Command, args []string) {
-
 		initAwsSess()
 
 		asgName := lib.GetAsgNameForEcsCluster(AwsSess, cluster)
 		if asgName == "" {
-			fmt.Println("Unable to find ASG name for ECS cluster ", cluster)
-			os.Exit(1)
+			log.Fatalln("Unable to find ASG name for ECS cluster", cluster)
 		}
 
 		instancesToTerminate := lib.GetInstanceListForAsg(AwsSess, asgName)
@@ -50,8 +49,7 @@ var replaceInstancesCmd = &cobra.Command{
 		for _, instanceID := range instancesToTerminate {
 			_, err := terminateInstance(*instanceID)
 			if err != nil {
-				fmt.Println("Unable to terminate instance: ", err)
-				os.Exit(1)
+				log.Fatalln("Unable to terminate instance:", err)
 			}
 			waitForZeroPendingTasks(cluster)
 		}

cmd/ecsRightSizeCluster.go

@@ -15,8 +15,11 @@
 package cmd
 
 import (
-	"github.com/silinternational/awsops/lib"
+	"log"
+
 	"github.com/spf13/cobra"
+
+	"github.com/silinternational/awsops/lib"
 )
 
 var atLeastServiceDesiredCount bool
@@ -33,7 +36,10 @@ support running all tasks with as few servers as is needed.
 This function may scale a cluster up or down depending on services.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		initAwsSess()
-		lib.RightSizeAsgForEcsCluster(AwsSess, cluster, atLeastServiceDesiredCount)
+		err := lib.RightSizeAsgForEcsCluster(AwsSess, cluster, atLeastServiceDesiredCount)
+		if err != nil {
+			log.Fatalln(err)
+		}
 	},
 }
 

cmd/lambdaInvoke.go

@@ -16,13 +16,17 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/silinternational/awsops/lib"
+	"log"
+
 	"github.com/spf13/cobra"
-	"os"
+
+	"github.com/silinternational/awsops/lib"
 )
 
-var functionName string
-var payload string
+var (
+	functionName string
+	payload      string
+)
 
 // invokeCmd represents the invoke command
 var invokeCmd = &cobra.Command{
@@ -34,8 +38,7 @@ var invokeCmd = &cobra.Command{
 
 		result, err := lib.LambdaInvoke(AwsSess, functionName, payload)
 		if err != nil {
-			fmt.Println(err.Error())
-			os.Exit(1)
+			log.Fatalln(err)
 		}
 
 		fmt.Printf("Response: [code: %v] %s", *result.StatusCode, result.Payload)

cmd/root.go

@@ -16,20 +16,22 @@ package cmd
 
 import (
 	"fmt"
-	"os"
+	"log"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
 )
 
-var AwsSess *session.Session
-var cfgFile string
-var Profile string
-var Region string
+var (
+	AwsSess *session.Session
+	cfgFile string
+	Profile string
+	Region  string
+)
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
@@ -45,8 +47,7 @@ var rootCmd = &cobra.Command{
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+		log.Fatalln(err)
 	}
 }
 
@@ -74,8 +75,7 @@ func initConfig() {
 		// Find home directory.
 		home, err := homedir.Dir()
 		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
+			log.Fatalln(err)
 		}
 
 		// Search config in home directory with name ".awsops" (without extension).
@@ -101,7 +101,7 @@ func initAwsSess() {
 		}))
 	} else {
 		AwsSess = session.Must(session.NewSession(&aws.Config{
-			Region:      aws.String(Region),
+			Region: aws.String(Region),
 		}))
 	}
 }

go.mod

@@ -8,16 +8,19 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/viper v1.15.0
+	github.com/stretchr/testify v1.8.1
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/afero v1.9.4 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect

lib/asg.go

@@ -2,8 +2,8 @@ package lib
 
 import (
 	"fmt"
+	"log"
 	"math"
-	"os"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -20,8 +20,7 @@ func GetAsgNameForEcsCluster(awsSess *session.Session, cluster string) string {
 		InstanceIds: instanceIDs,
 	})
 	if err != nil {
-		fmt.Println("Unable to get asg name from instance: ", err)
-		os.Exit(1)
+		log.Fatalln("Unable to get asg name from instance:", err)
 	}
 
 	for _, tag := range instanceDetails.Reservations[0].Instances[0].Tags {
@@ -45,8 +44,7 @@ func DetachAndReplaceAsgInstances(awsSess *session.Session, asgName string, inst
 		ShouldDecrementDesiredCapacity: &decrement,
 	})
 	if err != nil {
-		fmt.Println("Unable to detach instances: ", err)
-		os.Exit(1)
+		log.Fatalln("Unable to detach instances:", err)
 	}
 
 	fmt.Printf("done\n")
@@ -83,13 +81,11 @@ func GetInstanceTypeFromLaunchConfiguration(awsSess *session.Session, launchConf
 
 	lc, err := autoscaling.New(awsSess).DescribeLaunchConfigurations(input)
 	if err != nil {
-		fmt.Println("Unable to describe launch configuration: ", err.Error())
-		os.Exit(1)
+		log.Fatalln("Unable to describe launch configuration:", err)
 	}
 
 	if len(lc.LaunchConfigurations) != 1 {
-		fmt.Println("Expected one Launch Configuration, received ", len(lc.LaunchConfigurations))
-		os.Exit(1)
+		log.Fatalln("Expected one Launch Configuration, received", len(lc.LaunchConfigurations))
 	}
 
 	return *lc.LaunchConfigurations[0].InstanceType
@@ -106,13 +102,11 @@ func GetInstanceTypeFromLaunchTemplate(awsSess *session.Session, launchTemplateN
 
 	lt, err := ec2Client.DescribeLaunchTemplates(input)
 	if err != nil {
-		fmt.Println("Unable to describe Launch Template, err: ", err.Error())
-		os.Exit(1)
+		log.Fatalln("Unable to describe Launch Template, err:", err)
 	}
 
 	if len(lt.LaunchTemplates) != 1 {
-		fmt.Println("Expected one Launch Template, found ", len(lt.LaunchTemplates))
-		os.Exit(1)
+		log.Fatalln("Expected one Launch Template, found", len(lt.LaunchTemplates))
 	}
 
 	ltvInput := ec2.DescribeLaunchTemplateVersionsInput{
@@ -121,13 +115,11 @@ func GetInstanceTypeFromLaunchTemplate(awsSess *session.Session, launchTemplateN
 	}
 	ltv, err := ec2Client.DescribeLaunchTemplateVersions(&ltvInput)
 	if err != nil {
-		fmt.Println("Unable to describe Launch Template version, error: ", err.Error())
-		os.Exit(1)
+		log.Fatalln("Unable to describe Launch Template version, error:", err)
 	}
 
 	if len(ltv.LaunchTemplateVersions) != 1 {
-		fmt.Println(`Expected one "$Latest" Launch Template version, received `, len(lt.LaunchTemplates))
-		os.Exit(1)
+		log.Fatalln(`Expected one "$Latest" Launch Template version, received`, len(lt.LaunchTemplates))
 	}
 
 	return *ltv.LaunchTemplateVersions[0].LaunchTemplateData.InstanceType
@@ -144,26 +136,47 @@ func GetInstanceTypeForAsg(awsSess *session.Session, asgName string) string {
 		return GetInstanceTypeFromLaunchTemplate(awsSess, *asg.LaunchTemplate.LaunchTemplateName)
 	}
 
-	fmt.Println("Unable to determine the ASG instance type. No Launch Template nor Launch Configuration is defined.")
-	os.Exit(1)
+	log.Fatalln("Unable to determine the ASG instance type. No Launch Template nor Launch Configuration is defined.")
 	return ""
 }
 
-func HowManyServersNeededForAsg(serverType string, memory, cpu int64) int64 {
+// HowManyServersNeededForAsg computes the theoretical number of servers needed based on the total resources needed,
+// assuming near-perfect utilization of server resources. It does not take into account the "wasted" resources on an
+// individual server when the free resources are not sufficient to place any of the desired containers.
+func HowManyServersNeededForAsg(serverType string, resourcesNeeded ResourceSizes) int64 {
 	instanceSpecs, valid := InstanceTypes[serverType]
 	if !valid {
-		fmt.Println("Invalid server type provided: ", serverType)
-		os.Exit(1)
+		log.Fatalln("Invalid server type provided:", serverType)
 	}
 
-	neededForMem := math.Ceil(float64(memory) / float64(instanceSpecs.MemoryMb))
-	neededForCPU := math.Ceil(float64(cpu) / float64(instanceSpecs.CPUUnits))
+	if resourcesNeeded.LargestMemory > instanceSpecs.MemoryMb {
+		log.Fatalf("Configured instance type is not large enough. Available memory is %d, but largest task needs %d",
+			instanceSpecs.MemoryMb, resourcesNeeded.LargestMemory)
+	}
 
-	if neededForMem > neededForCPU {
-		return int64(neededForMem)
+	if resourcesNeeded.LargestCPU > instanceSpecs.CPUUnits {
+		log.Fatalf("Configured instance type is not large enough. Available CPU is %d, but largest task needs %d",
+			instanceSpecs.CPUUnits, resourcesNeeded.LargestCPU)
 	}
 
-	return int64(neededForCPU)
+	// Some memory in each instance cannot be used because no container can be placed in the last portion available.
+	// This assumes the best-case container placement.
+	usableMemory := max(1, instanceSpecs.MemoryMb-resourcesNeeded.SmallestMemory)
+	usableCPU := max(1, instanceSpecs.CPUUnits-resourcesNeeded.SmallestCPU)
+
+	neededForMem := divideAndRoundUp(resourcesNeeded.TotalMemory, usableMemory)
+	neededForCPU := divideAndRoundUp(resourcesNeeded.TotalCPU, usableCPU)
+
+	serversNeeded := max(neededForCPU, neededForMem)
+	if serversNeeded > 100 {
+		log.Fatalf("Calculated need of %d instances, which is over the predefined threshold. Exiting.", serversNeeded)
+	}
+
+	return serversNeeded
+}
+
+func divideAndRoundUp(numerator, divisor int64) int64 {
+	return int64(math.Ceil(float64(numerator) / float64(divisor)))
 }
 
 func GetAsgServerCount(awsSess *session.Session, asgName string) (desired int64, min int64, max int64) {
@@ -179,13 +192,11 @@ func GetAsg(awsSess *session.Session, asgName string) *autoscaling.Group {
 		AutoScalingGroupNames: []*string{&asgName},
 	})
 	if err != nil {
-		fmt.Println("Unable to get list of ASG groups: ", err)
-		os.Exit(1)
+		log.Fatalln("Unable to get list of ASG groups:", err)
 	}
 
 	if len(groups.AutoScalingGroups) != 1 {
-		fmt.Println("DescribeAutoScalingGroups did not return expected number of results. Expected: 1, Actual: ", len(groups.AutoScalingGroups))
-		os.Exit(1)
+		log.Fatalln("DescribeAutoScalingGroups did not return expected number of results. Expected: 1, Actual:", len(groups.AutoScalingGroups))
 	}
 
 	return groups.AutoScalingGroups[0]