FR: secrets import to services from AWS Secrets Manager (#142)

ecs_composex/common/settings.py

@@ -42,6 +42,7 @@
 from ecs_composex.utils.init_s3 import create_bucket
 from ecs_composex.ecs.ecs_service_config import set_service_ports
 from cfn_flip.yaml_dumper import LongCleanDumper
+from ecs_composex.secrets.secrets_config import parse_secrets
 
 
 def render_services_ports(services):
@@ -327,6 +328,7 @@ def set_content(self, kwargs, content=None):
         if keyisset("services", self.compose_content):
             render_services_ports(self.compose_content["services"])
         interpolate_env_vars(self.compose_content)
+        parse_secrets(self)
 
     def parse_command(self, kwargs, content=None):
         """

ecs_composex/ecs/ecs_container.py

@@ -32,7 +32,7 @@
 from ecs_composex.common import keyisset
 from ecs_composex.common.outputs import ComposeXOutput
 from ecs_composex.ecs import ecs_params
-from ecs_composex.ecs.ecs_container_config import import_env_variables
+from ecs_composex.ecs.ecs_container_config import import_env_variables, import_secrets
 
 
 class Container(object):
@@ -43,7 +43,7 @@ class Container(object):
     parameters = {}
     required_keys = ["image"]
 
-    def __init__(self, template, title, definition, config):
+    def __init__(self, template, title, definition, config, settings):
         """
 
         :param troposphere.Template template: template to add the container definition to
@@ -101,6 +101,7 @@ def __init__(self, template, title, definition, config):
             if isinstance(config.healthcheck, HealthCheck)
             else Ref(AWS_NO_VALUE),
         )
+        import_secrets(template, definition, self.definition, settings)
         values = []
         if isinstance(config.cpu_resa, int):
             values.append(("Cpu", "Cpu", str(config.cpu_resa)))

ecs_composex/ecs/ecs_container_config.py

@@ -23,7 +23,42 @@
 from troposphere import Ref, GetAtt, ImportValue, Sub
 from troposphere.ecs import ContainerDefinition, Environment
 
-from ecs_composex.common import LOG
+from ecs_composex.common import LOG, keyisset
+
+
+def import_secrets(template, definition, container, settings):
+    """
+    Function to import secrets from composex mapping to AWS Secrets in Secrets Manager
+
+    :param troposphere.Template template:
+    :param dict definition:
+    :param troposhere.ecs.ContainerDefinition container:
+    :param ecs_composex.common.settings.ComposeXSettings settings:
+    :return:
+    """
+    if keyisset("secrets", definition) and isinstance(definition["secrets"], list):
+        secrets = definition["secrets"]
+    else:
+        return
+    if not keyisset("secrets", settings.compose_content):
+        return
+    else:
+        settings_secrets = settings.compose_content["secrets"]
+    for secret in secrets:
+        if (
+            isinstance(secret, str)
+            and secret in settings_secrets
+            and keyisset("ComposeSecret", settings_secrets[secret])
+        ):
+            settings_secrets[secret]["ComposeSecret"].assign_to_task_definition(
+                template, container
+            )
+        elif isinstance(secret, dict) and keyisset("source", secret):
+            secret_name = secret["source"]
+            if keyisset("ComposeSecret", settings_secrets[secret_name]):
+                settings_secrets[secret_name][
+                    "ComposeSecret"
+                ].assign_to_task_definition(template, container)
 
 
 def import_env_variables(environment):

ecs_composex/ecs/ecs_service.py

@@ -170,16 +170,16 @@ class Task(object):
     Class to handle the Task definition building and parsing along with the service config.
     """
 
-    def __init__(self, template, containers_config, family_parameters):
+    def __init__(self, template, containers_config, family_parameters, settings):
         """
         Init method
         """
         self.family_config = None
         self.containers = []
         self.containers_config = containers_config
         self.stack_parameters = {}
-        self.sort_container_configs(template, containers_config)
         add_service_roles(template, self.family_config)
+        self.sort_container_configs(template, containers_config, settings)
         if self.family_config.use_xray:
             self.containers.append(define_xray_container())
             add_parameters(template, [ecs_params.XRAY_IMAGE])
@@ -214,7 +214,7 @@ def set_task_definition(self, template):
             ),
         )
 
-    def sort_container_configs(self, template, containers_config):
+    def sort_container_configs(self, template, containers_config, settings):
         """
         Method to sort out the containers dependencies and create the containers definitions based on the configs.
         :return:
@@ -236,6 +236,7 @@ def sort_container_configs(self, template, containers_config):
                 service_config["config"].resource_name,
                 service_config["definition"],
                 service_config["config"],
+                settings,
             )
             self.containers.append(container.definition)
             self.stack_parameters.update(container.stack_parameters)

ecs_composex/ecs/ecs_template.py

@@ -282,7 +282,7 @@ def handle_families_services(families, cluster_sg, settings):
                 "priority": 0,
                 "definition": service,
             }
-        task = Task(template, family_service_configs, family_parameters)
+        task = Task(template, family_service_configs, family_parameters, settings)
         family_parameters.update(task.stack_parameters)
         service = Service(
             template=template,

ecs_composex/secrets/SYNTAX.rst

@@ -0,0 +1,52 @@
+
+
+secrets
+=======
+
+.. seealso::
+
+    `docker-compose secrets reference`_
+
+You might have secrets in AWS Secrets Manager that you created outside of this application stack and your services
+need access to it.
+
+By defining secrets in docker-compose, you can do all of that work rather easily.
+To help make it as easy in AWS, simply set `external=True` and a few other settings to indicate how to get the secret.
+
+
+.. code-block:: yaml
+
+    version: "3.8"
+
+    services:
+      servicename:
+        image: abcd
+        secrets:
+          - abcd
+
+    secrets:
+      mysecret:
+        external: true
+        x-secret:
+          Name: /name/in/aws
+          LinkTo:
+            - EcsExecutionRole
+            - EcsTaskRole
+
+x-secret
+--------
+
+Name
+^^^^
+
+The name (also known as path) to the secret in AWS Secrets Manager.
+
+
+LinksTo
+^^^^^^^
+
+List to determine whether the TaskRole or ExecutionRole (or both) should have access to the Secret.
+If set as TaskRole, then the secret **value will not be exposed in env vars** and only the secret name will be set.
+
+
+.. _docker-compose secrets reference: https://docs.docker.com/compose/compose-file/#secrets

ecs_composex/secrets/__init__.py

@@ -0,0 +1,16 @@
+#  -*- coding: utf-8 -*-
+#   ECS ComposeX <https://github.com/lambda-my-aws/ecs_composex>
+#   Copyright (C) 2020  John Mille <[email protected]>
+#  #
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#  #
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#  #
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.

ecs_composex/secrets/secrets_aws.py

@@ -0,0 +1,16 @@
+#  -*- coding: utf-8 -*-
+#   ECS ComposeX <https://github.com/lambda-my-aws/ecs_composex>
+#   Copyright (C) 2020  John Mille <[email protected]>
+#  #
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#  #
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#  #
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.

ecs_composex/secrets/secrets_config.py

@@ -0,0 +1,121 @@
+#  -*- coding: utf-8 -*-
+#   ECS ComposeX <https://github.com/lambda-my-aws/ecs_composex>
+#   Copyright (C) 2020  John Mille <[email protected]>
+#  #
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#  #
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#  #
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Module to parse secrets from the compose content file.
+"""
+
+from troposphere import Sub, AWS_PARTITION, AWS_REGION, AWS_ACCOUNT_ID
+from troposphere.ecs import Secret as EcsSecret
+from troposphere.iam import Policy
+
+from ecs_composex.common import LOG, keyisset
+from ecs_composex.ecs.ecs_params import TASK_ROLE_T, EXEC_ROLE_T
+from ecs_composex.ecs.ecs_iam import define_service_containers
+from ecs_composex.ecs.ecs_container_config import extend_container_secrets
+
+
+RES_KEY = "secrets"
+XRES_KEY = "x-secrets"
+
+
+class ComposeSecret(object):
+    """
+    Class to represent a Compose secret.
+    """
+
+    def __init__(self, name, definition):
+        if not keyisset("Name", definition[XRES_KEY]):
+            raise KeyError(f"Missing Name in the {XRES_KEY} defintion")
+        self.name = name
+        aws_name = definition[XRES_KEY]["Name"]
+        if aws_name.startswith("arn:"):
+            self.aws_name = definition[XRES_KEY]["Name"]
+        else:
+            self.aws_name = Sub(
+                f"arn:${{{AWS_PARTITION}}}:secretsmanager:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}:secret:${aws_name}"
+            )
+        self.links = (
+            definition[XRES_KEY]["LinksTo"]
+            if keyisset("LinksTo", definition[XRES_KEY])
+            else [EXEC_ROLE_T]
+        )
+        self.ecs_secret = EcsSecret(Name=self.name, ValueFrom=self.aws_name)
+
+        self.validate_links()
+
+    def validate_links(self):
+        for link in self.links:
+            if not link in [EXEC_ROLE_T, TASK_ROLE_T]:
+                raise ValueError(
+                    "Links in LinksTo can only be one of",
+                    EXEC_ROLE_T,
+                    TASK_ROLE_T,
+                    "Got",
+                    link,
+                )
+
+    def assign_to_task_definition(self, template, container):
+        """
+        Method to add the secret to the given task definition
+
+        :param troposphere.Template template:
+        :param troposphere.ecs.ContainerDefinition container:
+        :return:
+        """
+        task_role = template.resources[TASK_ROLE_T]
+        exec_role = template.resources[EXEC_ROLE_T]
+        policy = Policy(
+            PolicyName=f"AccessSecret{self.name}",
+            PolicyDocument={
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Action": ["secretsmanager:GetSecretValue"],
+                        "Effect": "Allow",
+                        "Resource": self.aws_name,
+                        "Sid": f"AccessToSecret{self.name}",
+                    }
+                ],
+            },
+        )
+        if EXEC_ROLE_T in self.links and hasattr(exec_role, "Policies"):
+            exec_role.Policies.append(policy)
+        elif EXEC_ROLE_T in self.links and not hasattr(exec_role, "Policies"):
+            setattr(exec_role, "Policies", [policy])
+        if TASK_ROLE_T in self.links and hasattr(task_role, "Policies"):
+            task_role.Policies.append(policy)
+        elif TASK_ROLE_T in self.links and not hasattr(task_role, "Policies"):
+            setattr(task_role, "Policies", [policy])
+        extend_container_secrets(container, self.ecs_secret)
+
+
+def parse_secrets(settings):
+    """
+    Function to parse the settings compose content and define the secrets.
+
+    :param ecs_composex.common.settings.ComposeXSettings settings:
+    :return:
+    """
+    if not keyisset(RES_KEY, settings.compose_content):
+        return
+    secrets = settings.compose_content[RES_KEY]
+    for secret_name in secrets:
+        secret_def = secrets[secret_name]
+        if keyisset("external", secret_def):
+            LOG.info(f"Adding secret {secret_name} to settings")
+            secret_def["ComposeSecret"] = ComposeSecret(secret_name, secret_def)

features/use-cases/blog-all-features.yml

@@ -49,6 +49,8 @@ services:
       - app03:dateteller
 
   app02:
+    secrets:
+      - testsecret
     image: 373709687836.dkr.ecr.eu-west-1.amazonaws.com/blog-app-02:latest
     ports:
       - 5000
@@ -64,6 +66,9 @@ services:
 
   app03:
     image: 373709687836.dkr.ecr.eu-west-1.amazonaws.com/blog-app-02:latest
+    secrets:
+      - undefined
+      - source: testsecret
     ports:
       - 5000
     deploy:
@@ -336,3 +341,14 @@ x-cluster:
         Base: 2
       - CapacityProvider: FARGATE
         Weight: 1
+
+secrets:
+  testsecret:
+    external: true
+    x-secrets:
+      Name: /path/to/my/secret
+
+  testabcdsecret:
+    file: /dev/null
+    x-secrets:
+      Name: /nowhere

use-cases/blog.yml

@@ -40,6 +40,8 @@ services:
 
   app02:
     image: 373709687836.dkr.ecr.eu-west-1.amazonaws.com/blog-app-02:latest
+    secrets:
+      - testsecret
     ports:
       - 5000
     deploy:
@@ -81,3 +83,13 @@ services:
 x-tags:
   owner: johnpreston
   contact: [email protected]
+
+
+secrets:
+  testsecret:
+    external: true
+    x-secrets:
+      Name: /path/to/my/secret
+      LinksTo:
+        - EcsTaskRole
+        - EcsExecutionRole