feat(devnet): continuous integration gssmr devnet on AWS ECS

.github/workflows/devnet.yml

@@ -57,6 +57,12 @@ jobs:
           docker build -f=devnet/bob.Dockerfile . --progress=plain -t=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t=$ECR_REGISTRY/$ECR_REPOSITORY:latest --build-arg key=charlie --build-arg DD_API_KEY=$DD_API_KEY --build-arg METRICS_NAMESPACE=gossamer.ecs.devnet
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
+
+      - name: Scale down existing Bob and Charlie
+        id: scale-down
+        working-directory: ./devnet/gssmr-ecs
+        run: |
+          go run cmd/scale-down-ecs-service/main.go -c gssmr-ecs -s="gssmr-ecs-(Charlie|Bob)Service-.+$"
 
       - name: docker compose up
         id: docker-compose-up

devnet/cmd/scale-down-ecs-service/main.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"regexp"
+	"syscall"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/jessevdk/go-flags"
+)
+
+var (
+	svc *ecs.ECS
+)
+
+func init() {
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	}))
+	svc = ecs.New(sess)
+}
+
+func findServiceArns(ctx context.Context, cluster, serviceRegex string) (serviceArns []*string, err error) {
+	r, err := regexp.Compile(serviceRegex)
+	if err != nil {
+		return
+	}
+
+	var lsi = &ecs.ListServicesInput{
+		Cluster: &cluster,
+	}
+	for {
+		var lso *ecs.ListServicesOutput
+		lso, err = svc.ListServicesWithContext(ctx, lsi)
+		if err != nil {
+			return
+		}
+		for _, arn := range lso.ServiceArns {
+			if r.MatchString(*arn) {
+				serviceArns = append(serviceArns, arn)
+			}
+		}
+		if lso.NextToken != nil {
+			lsi.NextToken = lso.NextToken
+		} else {
+			break
+		}
+	}
+	return
+}
+
+func updateServices(ctx context.Context, cluster string, serviceArns []*string) (err error) {
+	for _, serviceArn := range serviceArns {
+		_, err = svc.UpdateServiceWithContext(ctx, &ecs.UpdateServiceInput{
+			Cluster:      &cluster,
+			Service:      serviceArn,
+			DesiredCount: aws.Int64(0),
+		})
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func waitForRunningCount(ctx context.Context, cluster string, serviceArns []*string) (err error) {
+	ticker := time.NewTicker(time.Second * 5)
+main:
+	for {
+		select {
+		case <-ticker.C:
+			var dso *ecs.DescribeServicesOutput
+			dso, err = svc.DescribeServicesWithContext(ctx, &ecs.DescribeServicesInput{
+				Cluster:  &cluster,
+				Services: serviceArns,
+			})
+			if err != nil {
+				break main
+			}
+			scaledDown := make(map[string]bool)
+			for _, service := range dso.Services {
+				if service.RunningCount != nil && *service.RunningCount == 0 {
+					scaledDown[*service.ServiceArn] = true
+				}
+			}
+			if len(scaledDown) == len(serviceArns) {
+				break main
+			}
+		case <-ctx.Done():
+			err = fmt.Errorf("aborting waiting, received done from context")
+			break main
+		}
+	}
+	return
+}
+
+func scaleServices(ctx context.Context, cluster string, servicesRegex string) (err error) {
+	serviceArns, err := findServiceArns(ctx, cluster, servicesRegex)
+	if err != nil {
+		return
+	}
+
+	err = updateServices(ctx, cluster, serviceArns)
+	if err != nil {
+		return
+	}
+
+	err = waitForRunningCount(ctx, cluster, serviceArns)
+	return
+}
+
+type options struct {
+	ServicesRegex string `short:"s" long:"services" description:"regex query used to match against AWS service names" required:"true"` //nolint:lll
+	Cluster       string `short:"c" long:"cluster" description:"ECS cluster name, must be exact match"`
+}
+
+func main() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+	var opts options
+	_, err := flags.Parse(&opts)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error)
+	go func() {
+		err = scaleServices(ctx, opts.Cluster, opts.ServicesRegex)
+		done <- err
+	}()
+
+	for {
+		select {
+		case err := <-done:
+			if err != nil {
+				log.Panic(err)
+			}
+			// happy path
+			return
+		case <-sigs:
+			cancel()
+		}
+	}
+}

devnet/cmd/scale-down-ecs-service/main_test.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"context"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+)
+
+func Test_findServiceArns(t *testing.T) {
+	type args struct {
+		cluster      string
+		serviceRegex string
+	}
+	tests := []struct {
+		name            string
+		args            args
+		wantServiceArns []string
+		wantErr         bool
+	}{
+		{
+			args: args{
+				cluster:      "gssmr-ecs",
+				serviceRegex: "gssmr-ecs-(Charlie|Bob)Service-.+$",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotServiceArns, err := findServiceArns(context.Background(), tt.args.cluster, tt.args.serviceRegex)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("findServiceArns() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotServiceArns, tt.wantServiceArns) {
+				t.Errorf("findServiceArns() = %v, want %v", gotServiceArns, tt.wantServiceArns)
+			}
+		})
+	}
+}
+
+func Test_updateServices(t *testing.T) {
+	type args struct {
+		cluster     string
+		serviceArns []*string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			args: args{
+				cluster: "gssmr-ecs",
+				serviceArns: []*string{
+					aws.String("arn:aws:ecs:us-east-2:500822580415:service/gssmr-ecs/gssmr-ecs-CharlieService-J30Gr963xo2y"),
+					aws.String("arn:aws:ecs:us-east-2:500822580415:service/gssmr-ecs/gssmr-ecs-BobService-tAQb9CsOMLx7"),
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := updateServices(context.Background(), tt.args.cluster, tt.args.serviceArns); (err != nil) != tt.wantErr {
+				t.Errorf("updateServices() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func Test_waitForRunningCount(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	type args struct {
+		ctx         context.Context
+		cluster     string
+		serviceArns []*string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+		sidecar func(t *testing.T)
+	}{
+		{
+			args: args{
+				cluster: "gssmr-ecs",
+				serviceArns: []*string{
+					aws.String("arn:aws:ecs:us-east-2:500822580415:service/gssmr-ecs/gssmr-ecs-CharlieService-J30Gr963xo2y"),
+					aws.String("arn:aws:ecs:us-east-2:500822580415:service/gssmr-ecs/gssmr-ecs-BobService-tAQb9CsOMLx7"),
+				},
+				ctx: ctx,
+			},
+			sidecar: func(t *testing.T) {
+				<-time.After(6 * time.Second)
+				cancel()
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.sidecar != nil {
+				go tt.sidecar(t)
+			}
+			if err := waitForRunningCount(tt.args.ctx, tt.args.cluster, tt.args.serviceArns); (err != nil) != tt.wantErr {
+				t.Errorf("waitUntilScaledDown() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

go.mod

@@ -49,6 +49,7 @@ require (
 require (
 	github.com/ChainSafe/log15 v1.0.0 // indirect
 	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
+	github.com/aws/aws-sdk-go v1.42.20 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd v0.22.0-beta // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
@@ -85,6 +86,7 @@ require (
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/jcelliott/lumber v0.0.0-20160324203708-dd349441af25 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/klauspost/compress v1.12.3 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect

go.sum

@@ -106,6 +106,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.42.20 h1:nQkkmTWK5N2Ao1iVzoOx1HTIxwbSWErxyZ1eiwLJWc4=
+github.com/aws/aws-sdk-go v1.42.20/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo=
 github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y=
@@ -582,6 +584,7 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M
 github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -1476,6 +1479,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=