feat(devnet): continuous integration gssmr devnet on AWS ECS

.github/workflows/devnet.yml

@@ -0,0 +1,74 @@
+name: devnet
+on:
+  push:
+    branches:
+      - devnet
+      - tim/ecs_devnet
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push alice image to Amazon ECR
+        id: build-alice
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: gssmr-devnet-alice
+          IMAGE_TAG: ${{ github.sha }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+        run: |
+          docker build -f=devnet/alice.Dockerfile . --progress=plain -t=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t=$ECR_REGISTRY/$ECR_REPOSITORY:latest --build-arg DD_API_KEY=$DD_API_KEY --build-arg METRICS_NAMESPACE=gossamer.ecs.devnet
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
+
+      - name: Build, tag, and push bob image to Amazon ECR
+        id: build-bob
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: gssmr-devnet-bob
+          IMAGE_TAG: ${{ github.sha }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+        run: |
+          docker build -f=devnet/bob.Dockerfile . --progress=plain -t=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t=$ECR_REGISTRY/$ECR_REPOSITORY:latest --build-arg key=bob --build-arg DD_API_KEY=$DD_API_KEY --build-arg METRICS_NAMESPACE=gossamer.ecs.devnet
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
+
+      - name: Build, tag, and push charlie image to Amazon ECR
+        id: build-charlie
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: gssmr-devnet-charlie
+          IMAGE_TAG: ${{ github.sha }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+        run: |
+          docker build -f=devnet/bob.Dockerfile . --progress=plain -t=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t=$ECR_REGISTRY/$ECR_REPOSITORY:latest --build-arg key=charlie --build-arg DD_API_KEY=$DD_API_KEY --build-arg METRICS_NAMESPACE=gossamer.ecs.devnet
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
+
+      - name: Scale down existing Bob and Charlie
+        id: scale-down
+        working-directory: ./devnet
+        run: |
+          go run cmd/scale-down-ecs-service/main.go -c gssmr-ecs -s="gssmr-ecs-(Charlie|Bob)Service-.+$"
+
+      - name: docker compose up
+        id: docker-compose-up
+        working-directory: ./devnet/gssmr-ecs
+        run: |
+          curl -L https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh
+          docker context create ecs gssmr-ecs --from-env
+          docker context use gssmr-ecs
+          docker compose up

.gitignore

@@ -15,6 +15,7 @@
 *.key
 !test-key.key
 !test-key-password.key
+!devnet/alice.node.key
 *.wasm
 
 test_data

cmd/gossamer/config.go

@@ -678,6 +678,11 @@ func setDotNetworkConfig(ctx *cli.Context, tomlCfg ctoml.NetworkConfig, cfg *dot
 		cfg.PublicIP = pubip
 	}
 
+	// check --pubdns flag and update node configuration
+	if pubdns := ctx.GlobalString(PublicDNSFlag.Name); pubdns != "" {
+		cfg.PublicDNS = pubdns
+	}
+
 	if len(cfg.PersistentPeers) == 0 {
 		cfg.PersistentPeers = []string(nil)
 	}

cmd/gossamer/config_test.go

@@ -531,6 +531,21 @@ func TestNetworkConfigFromFlags(t *testing.T) {
 				PublicIP:          "10.0.5.2",
 			},
 		},
+		{
+			"Test gossamer --pubdns",
+			[]string{"config", "pubdns"},
+			[]interface{}{testCfgFile.Name(), "alice"},
+			dot.NetworkConfig{
+				Port:              testCfg.Network.Port,
+				Bootnodes:         testCfg.Network.Bootnodes,
+				ProtocolID:        testCfg.Network.ProtocolID,
+				NoBootstrap:       testCfg.Network.NoBootstrap,
+				NoMDNS:            false,
+				DiscoveryInterval: time.Second * 10,
+				MinPeers:          testCfg.Network.MinPeers,
+				PublicDNS:         "alice",
+			},
+		},
 	}
 
 	for _, c := range testcases {

cmd/gossamer/flags.go

@@ -223,6 +223,11 @@ var (
 		Name:  "pubip",
 		Usage: "Overrides public IP address used for peer to peer networking",
 	}
+	// PublicDNSFlag uses the supplied DNS for broadcasting
+	PublicDNSFlag = cli.StringFlag{
+		Name:  "pubdns",
+		Usage: "Overrides public DNS used for peer to peer networking",
+	}
 )
 
 // RPC service configuration flags
@@ -414,6 +419,7 @@ var (
 		NoBootstrapFlag,
 		NoMDNSFlag,
 		PublicIPFlag,
+		PublicDNSFlag,
 
 		// rpc flags
 		RPCEnabledFlag,

devnet/README.md

@@ -51,4 +51,25 @@ This key is injected in `alice.Dockerfile` so it uses the same public key for th
 
 ### docker-compose.yml
 
-The Docker Compose file.  Specifies the IP addresses of all the nodes.  
+The Docker Compose file.  Specifies the IP addresses of all the nodes. 
+
+
+## ECS Fargate Deployment
+
+The `docker-compose.yml` file within `devnet/gssmr-ecs` folder uses Docker Compose ECS plugin to deploy and update an existing AWS ECS Cluster using the Fargate launch type running a Gossamer devnet with 3 services corresponding to the 3 keys used `alice`, `bob`, and `charlie`.  
+
+### Deployment
+
+Currently deployment is handled via a github workflow.  Pushing the `devnet` branch will initiate the deploy process.  Steps are outlined in `/.github/workflows/devnet.yml`. 
+
+At a high level, images for the `alice`, `bob` and `charlie` correspond to ECS services under the same name.  The docker images are built based on the latest commit on the `devnet` branch.  These images are pushed to ECR.  A specific type of Docker context is required to use the ECS plugin.  Deploying and updating is as simple as:
+
+```
+docker context create ecs gssmr-ecs --from-env
+docker context use gssmr-ecs
+docker compose up
+```
+### Prometheus to Datadog
+
+Prometheus metrics are automatically piped to Datadog.  All metrics from the ECS devnet are prefixed with `gossamer.ecs.devnet`.  
+

devnet/alice.Dockerfile

@@ -4,7 +4,8 @@
 FROM golang:1.17
 
 ARG DD_API_KEY=somekey
-RUN DD_API_KEY=${DD_API_KEY}  DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
+ENV DD_API_KEY=${DD_API_KEY}
+RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
 
 WORKDIR /gossamer
 
@@ -23,7 +24,10 @@ RUN gossamer --key=alice init
 # use a hardcoded key for alice, so we can determine what the peerID is for subsequent nodes
 RUN cp devnet/alice.node.key ~/.gossamer/gssmr/node.key
 
-RUN go run devnet/cmd/update-dd-agent-confd/main.go -n=gossamer.local.devnet -t=key:alice > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml
+ARG METRICS_NAMESPACE=gossamer.local.devnet
+
+RUN go run devnet/cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:alice > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml
+
+ENTRYPOINT service datadog-agent start && gossamer --key=alice --babe-lead --publish-metrics --rpc --rpc-external=true --pubdns=alice
 
-ENTRYPOINT service datadog-agent start && gossamer --key=alice --babe-lead --publish-metrics --rpc --rpc-external=true --pubip=10.5.0.2
 EXPOSE 7001/tcp 8545/tcp 8546/tcp 8540/tcp 9876/tcp

devnet/alice.node.key

@@ -0,0 +1 @@
+93ce444331ced4d2f7bfb8296267544e20c2591dbf310c7ea3af672f2879cf8fa999ca3052b725123c2939085e3158dba26624fcfd05c361b1adcc8b27785e35

devnet/bob.Dockerfile

@@ -4,7 +4,8 @@
 FROM golang:1.17
 
 ARG DD_API_KEY=somekey
-RUN DD_API_KEY=${DD_API_KEY} DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
+ENV DD_API_KEY=${DD_API_KEY}
+RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
 
 WORKDIR /gossamer
 
@@ -20,16 +21,14 @@ RUN cp -f devnet/chain/gssmr/genesis-spec.json chain/gssmr/genesis-spec.json
 
 ARG key
 RUN test -n "$key"
-ARG pubip
-RUN test -n "$pubip"
-
 ENV key=${key}
-ENV pubip=${pubip}
 
 RUN gossamer --key=${key} init
 
-RUN go run devnet/cmd/update-dd-agent-confd/main.go -n=gossamer.local.devnet -t=key:${key} > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml
+ARG METRICS_NAMESPACE=gossamer.local.devnet
+
+RUN go run devnet/cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:${key} > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml
 
-ENTRYPOINT service datadog-agent start && gossamer --key=${key} --bootnodes=/ip4/10.5.0.2/tcp/7001/p2p/12D3KooWMER5iow67nScpWeVqEiRRx59PJ3xMMAYPTACYPRQbbWU --publish-metrics --rpc --pubip=${pubip}
+ENTRYPOINT service datadog-agent start && gossamer --key=${key} --bootnodes=/dns/alice/tcp/7001/p2p/12D3KooWMER5iow67nScpWeVqEiRRx59PJ3xMMAYPTACYPRQbbWU --publish-metrics --rpc --pubdns=${key}
 
 EXPOSE 7001/tcp 8545/tcp 8546/tcp 8540/tcp 9876/tcp

devnet/chain/gssmr/genesis-spec.json

@@ -28,6 +28,14 @@
 					[
 						"5DFNv4Txc4b88qHqQ6GG4D646QcT4fN3jjS2G3r1PyZkfDut",
 						1
+					],
+					[
+						"5FRbF5E5xpBxaCdjuVkCAtoLgvvoL6wJoE7kAVTUdZBwpgi7",
+						1
+					],
+					[
+						"5DYo8CvjQcBQFdehVhansDiZCPebpgqvNC8PQPi6K9cL9giT",
+						1
 					]
 				]
 			},

devnet/cmd/scale-down-ecs-service/main.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"regexp"
+	"syscall"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/jessevdk/go-flags"
+)
+
+var (
+	svc *ecs.ECS
+)
+
+func init() {
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	}))
+	svc = ecs.New(sess)
+}
+
+func findServiceArns(ctx context.Context, cluster, serviceRegex string) (serviceArns []*string, err error) {
+	r, err := regexp.Compile(serviceRegex)
+	if err != nil {
+		return
+	}
+
+	var lsi = &ecs.ListServicesInput{
+		Cluster: &cluster,
+	}
+	for {
+		var lso *ecs.ListServicesOutput
+		lso, err = svc.ListServicesWithContext(ctx, lsi)
+		if err != nil {
+			return
+		}
+		for _, arn := range lso.ServiceArns {
+			if r.MatchString(*arn) {
+				serviceArns = append(serviceArns, arn)
+			}
+		}
+		if lso.NextToken != nil {
+			lsi.NextToken = lso.NextToken
+		} else {
+			break
+		}
+	}
+	return
+}
+
+func updateServices(ctx context.Context, cluster string, serviceArns []*string) (err error) {
+	for _, serviceArn := range serviceArns {
+		_, err = svc.UpdateServiceWithContext(ctx, &ecs.UpdateServiceInput{
+			Cluster:      &cluster,
+			Service:      serviceArn,
+			DesiredCount: aws.Int64(0),
+		})
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func waitForRunningCount(ctx context.Context, cluster string, serviceArns []*string) (err error) {
+	ticker := time.NewTicker(time.Second * 5)
+main:
+	for {
+		select {
+		case <-ticker.C:
+			var dso *ecs.DescribeServicesOutput
+			dso, err = svc.DescribeServicesWithContext(ctx, &ecs.DescribeServicesInput{
+				Cluster:  &cluster,
+				Services: serviceArns,
+			})
+			if err != nil {
+				break main
+			}
+			scaledDown := make(map[string]bool)
+			for _, service := range dso.Services {
+				if service.RunningCount != nil && *service.RunningCount == 0 {
+					scaledDown[*service.ServiceArn] = true
+				}
+			}
+			if len(scaledDown) == len(serviceArns) {
+				break main
+			}
+		case <-ctx.Done():
+			err = fmt.Errorf("aborting waiting, received done from context")
+			break main
+		}
+	}
+	return
+}
+
+func scaleServices(ctx context.Context, cluster string, servicesRegex string) (err error) {
+	serviceArns, err := findServiceArns(ctx, cluster, servicesRegex)
+	if err != nil {
+		return
+	}
+
+	err = updateServices(ctx, cluster, serviceArns)
+	if err != nil {
+		return
+	}
+
+	err = waitForRunningCount(ctx, cluster, serviceArns)
+	return
+}
+
+type options struct {
+	ServicesRegex string `short:"s" long:"services" description:"regex query used to match against AWS service names" required:"true"` //nolint:lll
+	Cluster       string `short:"c" long:"cluster" description:"ECS cluster name, must be exact match"`
+}
+
+func main() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+	var opts options
+	_, err := flags.Parse(&opts)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error)
+	go func() {
+		err = scaleServices(ctx, opts.Cluster, opts.ServicesRegex)
+		done <- err
+	}()
+
+	for {
+		select {
+		case err := <-done:
+			if err != nil {
+				log.Panic(err)
+			}
+			// happy path
+			return
+		case <-sigs:
+			cancel()
+		}
+	}
+}