From 9de513921a8fef262bde79ccadddff4aa0468025 Mon Sep 17 00:00:00 2001 From: John Preston Date: Wed, 5 Aug 2020 16:02:34 +0100 Subject: [PATCH 01/13] WIP x-s3 --- ecs_composex/s3/__init__.py | 23 +++++ ecs_composex/s3/s3_aws.py | 20 ++++ ecs_composex/s3/s3_ecs.py | 35 +++++++ ecs_composex/s3/s3_params.py | 27 ++++++ ecs_composex/s3/s3_perms.json | 27 ++++++ ecs_composex/s3/s3_stack.py | 84 +++++++++++++++++ ecs_composex/s3/s3_template.py | 162 +++++++++++++++++++++++++++++++++ use-cases/rds/rds_basic.yml | 34 +------ 8 files changed, 383 insertions(+), 29 deletions(-) create mode 100644 ecs_composex/s3/__init__.py create mode 100644 ecs_composex/s3/s3_aws.py create mode 100644 ecs_composex/s3/s3_ecs.py create mode 100644 ecs_composex/s3/s3_params.py create mode 100644 ecs_composex/s3/s3_perms.json create mode 100644 ecs_composex/s3/s3_stack.py create mode 100644 ecs_composex/s3/s3_template.py diff --git a/ecs_composex/s3/__init__.py b/ecs_composex/s3/__init__.py new file mode 100644 index 000000000..04778b0f5 --- /dev/null +++ b/ecs_composex/s3/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# ECS ComposeX +# Copyright (C) 2020 John Mille +# # +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from ecs_composex import __version__ as version + +metadata = { + "Type": "ComposeX", + "Properties": {"ecs_composex::module": "ecs_composex.s3", "Version": version}, +} diff --git a/ecs_composex/s3/s3_aws.py b/ecs_composex/s3/s3_aws.py new file mode 100644 index 000000000..2dc8dd3a9 --- /dev/null +++ b/ecs_composex/s3/s3_aws.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ECS ComposeX +# Copyright (C) 2020 John Mille +# # +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Functions to find buckets and identify settings about these. +""" diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py new file mode 100644 index 000000000..34d1330b7 --- /dev/null +++ b/ecs_composex/s3/s3_ecs.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# ECS ComposeX +# Copyright (C) 2020 John Mille +# # +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Functions to pass permissions to Services to access S3 buckets. +""" + + +def s3_to_ecs(queues, services_stack, services_families, res_root_stack, settings, **kwargs): + """ + Function to handle permissions assignment to ECS services. + + :param queues: x-sqs queues defined in compose file + :param ecs_composex.common.stack.ComposeXStack services_stack: services root stack + :param services_families: services families + :param ecs_composex.common.stack.ComposeXStack res_root_stack: s3 root stack + :param ecs_composex.common.settings.ComposeXSettings settings: ComposeX Settings for execution + :param dict kwargs: + :return: + """ + diff --git a/ecs_composex/s3/s3_params.py b/ecs_composex/s3/s3_params.py new file mode 100644 index 000000000..aff3d4b93 --- /dev/null +++ b/ecs_composex/s3/s3_params.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# ECS ComposeX +# Copyright (C) 2020 John Mille +# # +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from os import path +from ecs_composex.common.ecs_composex import X_KEY +from troposphere import Parameter + +RES_KEY = f"{X_KEY}{path.basename(path.dirname(path.abspath(__file__)))}" + +S3_BUCKET_NAME_T = "BucketName" +S3_BUCKET_NAME = Parameter( + S3_BUCKET_NAME_T, Type="String", AllowedPattern=r"^[a-z0-9-.]+$" +) diff --git a/ecs_composex/s3/s3_perms.json b/ecs_composex/s3/s3_perms.json new file mode 100644 index 000000000..5f37645e3 --- /dev/null +++ b/ecs_composex/s3/s3_perms.json @@ -0,0 +1,27 @@ +{ + "RWObjects": { + "Action": [ + "s3:GetObject*", + "s3:PutObject*" + ], + "Effect": "Allow" + }, + "ReadOnlyObjects": { + "Action": [ + "s3:GetObject*" + ], + "Effect": "Allow" + }, + "WriteOnlyObjects": { + "Action": [ + "s3:PutObject*" + ], + "Effect": "Allow" + }, + "PowerUser": { + "NotAction": [ + "s3:CreateBucket", + "s3:DeleteBucket" + ] + } +} diff --git a/ecs_composex/s3/s3_stack.py b/ecs_composex/s3/s3_stack.py new file mode 100644 index 000000000..cf3d484c8 --- /dev/null +++ b/ecs_composex/s3/s3_stack.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +# ECS ComposeX +# Copyright (C) 2020 John Mille +# # +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Module to control S3 stack +""" + +from troposphere import Ref, GetAtt +from troposphere.s3 import Bucket + +from ecs_composex.common import LOG, keyisset, build_template, NONALPHANUM +from ecs_composex.common.stacks import ComposeXStack +from ecs_composex.common.outputs import ComposeXOutput +from ecs_composex.s3.s3_params import RES_KEY, S3_BUCKET_NAME_T +from ecs_composex.s3.s3_template import generate_bucket + + +CFN_MAX_OUTPUTS = 50 + + +def create_s3_template(settings): + """ + Function to create the root S3 template. + + :param ecs_composex.common.settings.ComposeXSettings settings: + :return: + """ + mono_template = False + if not keyisset(RES_KEY, settings.compose_content): + return None + buckets = settings.compose_content[RES_KEY] + if len(list(buckets.keys())) <= CFN_MAX_OUTPUTS: + mono_template = True + + template = build_template(f"S3 root by ECS ComposeX for {settings.name}") + for bucket_name in buckets: + bucket_res_name = NONALPHANUM.sub("", bucket_name) + bucket = generate_bucket( + bucket_name, bucket_res_name, buckets[bucket_name], settings + ) + if bucket: + values = [ + (S3_BUCKET_NAME_T, "Arn", GetAtt(bucket, "Arn")), + (S3_BUCKET_NAME_T, "Name", Ref(bucket)), + ] + outputs = ComposeXOutput(bucket, values, True) + if mono_template: + template.add_resource(bucket) + template.add_output(outputs.outputs) + elif not mono_template: + bucket_template = build_template( + f"Template for DynamoDB bucket {bucket.title}" + ) + bucket_template.add_resource(bucket) + bucket_template.add_output(outputs.outputs) + bucket_stack = ComposeXStack( + bucket_res_name, stack_template=bucket_template + ) + template.add_resource(bucket_stack) + return template + + +class XResource(ComposeXStack): + """ + Class to handle S3 buckets + """ + + def __init__(self, title, settings, **kwargs): + stack_template = create_s3_template(settings) + super().__init__(title, stack_template, **kwargs) diff --git a/ecs_composex/s3/s3_template.py b/ecs_composex/s3/s3_template.py new file mode 100644 index 000000000..ac6f6b044 --- /dev/null +++ b/ecs_composex/s3/s3_template.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +# ECS ComposeX +# Copyright (C) 2020 John Mille +# # +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from troposphere import s3 +from troposphere import Ref, GetAtt, Sub +from troposphere import AWS_NO_VALUE + +from ecs_composex.common import LOG, keyisset +from ecs_composex.s3 import metadata +from ecs_composex.s3.s3_params import S3_BUCKET_NAME + + +def create_bucket_encryption_default(props=None): + if props is None: + props = {"SSEAlgorithm": "AES256"} + default_encryption = s3.ServerSideEncryptionByDefault(**props) + return s3.BucketEncryption( + ServerSideEncryptionConfiguration=[ + s3.ServerSideEncryptionRule( + ServerSideEncryptionByDefault=default_encryption + ) + ] + ) + + +def handle_bucket_encryption(properties, settings): + """ + Function to handle the S3 bucket encryption. + + :param dict properties: + :param dict settings: + :return: + """ + default = create_bucket_encryption_default() + if not keyisset("BucketEncryption", properties) and not keyisset( + "BucketEncryption", settings + ): + return default + elif keyisset("BucketEncryption", settings): + if ( + keyisset("SSEAlgorithm", settings["BucketEncryption"]) + and settings["BucketEncryption"]["SSEAlgorithm"] == "AES256" + ): + return default + elif ( + keyisset("SSEAlgorithm", settings["BucketEncryption"]) + and settings["BucketEncryption"]["SSEAlgorithm"] == "aws:kms" + ): + if not keyisset("KMSMasterKeyID", settings["BucketEncryption"]): + raise KeyError("Missing attribute KMSMasterKeyID for KMS Encryption") + else: + return create_bucket_encryption_default(settings["BucketEncryption"]) + + +def define_public_block_access(properties): + """ + Function to define the block access. + :param properties: + :return: + """ + + +def define_accelerate_config(properties, settings): + """ + Function to define AccelerateConfiguration + + :param properties: + :param settings: + :return: + """ + config = s3.AccelerateConfiguration(AccelerationStatus="SUSPENDED") + if keyisset("AccelerateConfiguration", properties): + config = s3.AccelerateConfiguration( + AccelerationStatus="SUSPENDED" + if not keyisset("AccelerationStatus", properties["AccelerateConfiguration"]) + else properties["AccelerateConfiguration"]["AccelerationStatus"] + ) + elif keyisset("AccelerationStatus", settings): + config = s3.AccelerateConfiguration( + AccelerationStatus=settings["AccelerationStatus"] + ) + return config + + +def define_bucket(bucket_name, res_name, definition): + """ + Function to generate the S3 bucket object + + :param bucket_name: + :param res_name: + :param definition: + :return: + """ + properties = definition["Properties"] if keyisset("Properties", definition) else {} + settings = definition["Settings"] if keyisset("Settings", definition) else {} + props = { + "AccelerateConfiguration": define_accelerate_config(properties, settings), + "AccessControl": s3.BucketOwnerFullControl + if not keyisset("AccessControl", properties) + else properties["AccessControl"], + "BucketEncryption": handle_bucket_encryption(definition, settings), + "BucketName": bucket_name, + "ObjectLockEnabled": False + if not keyisset("ObjectLockEnabled", properties) + else properties["ObjectLockEnabled"], + "PublicAccessBlockConfiguration": s3.PublicAccessBlockConfiguration( + **properties["PublicAccessBlockConfiguration"] + ) + if keyisset("PublicAccessBlockConfiguration", properties) + else s3.PublicAccessBlockConfiguration( + BlockPublicAcls=True, + BlockPublicPolicy=True, + IgnorePublicAcls=True, + RestrictPublicBuckets=True, + ), + "VersioningConfiguration": s3.VersioningConfiguration( + Status=properties["VersioningConfiguration"]["Status"] + ) + if keyisset("VersioningConfiguration", properties) + else Ref(AWS_NO_VALUE), + "WebsiteConfiguration": s3.WebsiteConfiguration, + "Metadata": metadata, + } + bucket = s3.Bucket(res_name, **props) + return bucket + + +def generate_bucket(bucket_name, bucket_res_name, bucket_definition, settings): + """ + Function to identify whether create new bucket or lookup for existing bucket + + :param bucket_name: + :param bucket_res_name: + :param bucket_definition: + :param settings: + :return: + """ + if keyisset("Lookup", bucket_definition): + LOG.info("If bucket is found, its ARN will be added to the task") + return + elif keyisset("Use", bucket_definition): + LOG.info(f"Assuming bucket {bucket_name} exists to use.") + return + if not keyisset("Properties", bucket_definition): + LOG.warning(f"Properties for bucket {bucket_name} were not defined. Skipping") + return + bucket = define_bucket(bucket_name, bucket_res_name, bucket_definition) + return bucket diff --git a/use-cases/rds/rds_basic.yml b/use-cases/rds/rds_basic.yml index f09d0a81d..a86da939c 100644 --- a/use-cases/rds/rds_basic.yml +++ b/use-cases/rds/rds_basic.yml @@ -7,8 +7,9 @@ services: image: 373709687836.dkr.ecr.eu-west-1.amazonaws.com/blog-app-01-rproxy:latest ports: - 80:80 + - 443:443 deploy: - replicas: 2 + replicas: 6 resources: reservations: cpus: "0.1" @@ -24,42 +25,17 @@ services: lb_type: application depends_on: - app - app: - image: 373709687836.dkr.ecr.eu-west-1.amazonaws.com/blog-app-01:xray - ports: - - 5000 - deploy: - resources: - reservations: - cpus: "0.25" - memory: "64M" - limits: - cpus: "0.5" - memory: "128M" - labels: - ecs.task.family: app01 - environment: - LOGLEVEL: DEBUG - x-configs: - use_xray: True - - backend: - image: nginx - ports: - - 80 x-rds: dbA: Properties: - Engine: "aurora-mysql" - EngineVersion: "5.7.12" + Engine: "aurora-postgresql" + EngineVersion: "11.7" Settings: EnvNames: - DBA Services: - - name: backend - access: RW - - name: app + - name: app01 access: RW dbB: From b3b185d4a8e2fcf2f6a26aabc6da1b0a01d6cf7c Mon Sep 17 00:00:00 2001 From: John Preston <1236150+JohnPreston@users.noreply.github.com> Date: Mon, 5 Oct 2020 08:22:30 +0100 Subject: [PATCH 02/13] WIP with lookup for S3 --- ecs_composex/common/aws.py | 17 ++++ ecs_composex/ecs_composex.py | 2 + ecs_composex/resource_settings.py | 69 +++++++++++++- ecs_composex/s3/s3_aws.py | 54 +++++++++++ ecs_composex/s3/s3_ecs.py | 101 ++++++++++++++++++++- ecs_composex/s3/s3_params.py | 4 + ecs_composex/s3/s3_perms.py | 101 +++++++++++++++++++++ ecs_composex/s3/s3_template.py | 15 ++- use-cases/s3/lookup_use_create_buckets.yml | 19 ++++ use-cases/s3/simple_s3_bucket.yml | 9 ++ 10 files changed, 382 insertions(+), 9 deletions(-) create mode 100644 ecs_composex/s3/s3_perms.py create mode 100644 use-cases/s3/lookup_use_create_buckets.yml create mode 100644 use-cases/s3/simple_s3_bucket.yml diff --git a/ecs_composex/common/aws.py b/ecs_composex/common/aws.py index 7db5ddbc0..4b73c3b98 100644 --- a/ecs_composex/common/aws.py +++ b/ecs_composex/common/aws.py @@ -176,6 +176,23 @@ def find_aws_resource_arn_from_tags_api( return handle_search_results(arns, name, res_types, res_type, service_code) +def define_tagsgroups_filter_tags(tags): + """ + Function to create the filters out of tags list + + :param list tags: list of Key/Value dict + :return: filters + :rtype: list + """ + filters = [] + for tag in tags: + key = list(tag.keys())[0] + filter_name = key + filter_value = tag[key] + filters.append({"Key": filter_name, "Values": (filter_value,)}) + return filters + + def get_region_azs(session): """Function to return the AZ from a given region. Uses default region for this diff --git a/ecs_composex/ecs_composex.py b/ecs_composex/ecs_composex.py index 99cd3355c..77a0b968d 100644 --- a/ecs_composex/ecs_composex.py +++ b/ecs_composex/ecs_composex.py @@ -77,6 +77,8 @@ "dynamodb", f"{X_KEY}kms", "kms", + f"{X_KEY}s3", + "s3", ] EXCLUDED_X_KEYS = [ f"{X_KEY}configs", diff --git a/ecs_composex/resource_settings.py b/ecs_composex/resource_settings.py index 338249454..024284809 100644 --- a/ecs_composex/resource_settings.py +++ b/ecs_composex/resource_settings.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # ECS ComposeX # Copyright (C) 2020 John Mille # # @@ -77,3 +77,70 @@ def generate_resource_permissions(resource_name, policies, attribute, arn=None): PolicyDocument=clean_policy, ) return resource_policies + + +def generate_resource_envvars(resource_name, resource, attribute, arn=None): + """ + Function to generate environment variables that can be added to a container definition + shall the ecs_service need to know about the Queue + + :param str resource_name: The name of the resource + :param dict resource: The resource definition as defined in docker-compose file. + :param str attribute: the attribute of the resource we are using for Import + :param str arn: The ARN of the resource if already looked up. + + :return: environment key/pairs + :rtype: list + """ + env_names = [] + export_strings = ( + generate_export_strings(resource_name, attribute) if not arn else arn + ) + if keyisset("Settings", resource) and keyisset("EnvNames", resource["Settings"]): + for env_name in resource["Settings"]["EnvNames"]: + env_names.append( + Environment( + Name=env_name, + Value=export_strings, + ) + ) + if resource_name not in resource["Settings"]["EnvNames"]: + env_names.append( + Environment( + Name=resource_name, + Value=export_strings, + ) + ) + else: + env_names.append( + Environment( + Name=resource_name, + Value=export_strings, + ) + ) + return env_names + + +def validate_lookup_resource(resource_name, resource_def, res_root_stack): + """ + Function to validate a resource has attributes to lookup. + + :param str resource_name: + :param dict resource_def: + :param ecs_composex.common.stacks.ComposeXStack res_root_stack: + :return: + """ + if resource_name not in res_root_stack.stack_template.resources and not keyisset( + "Lookup", resource_def + ): + raise KeyError( + f"{resource_name} is not created in ComposeX and does not have Lookup attribute" + ) + if ( + not resource_name in res_root_stack.stack_template.resources + and keyisset("Lookup", resource_def) + and not keyisset("Tags", resource_def["Lookup"]) + ): + raise KeyError( + f"{resource_name} is defined for lookup but there are no tags indicated." + ) diff --git a/ecs_composex/s3/s3_aws.py b/ecs_composex/s3/s3_aws.py index 2dc8dd3a9..1ce6a2d87 100644 --- a/ecs_composex/s3/s3_aws.py +++ b/ecs_composex/s3/s3_aws.py @@ -18,3 +18,57 @@ """ Functions to find buckets and identify settings about these. """ + +import re + +from botocore.exceptions import ClientError + +from ecs_composex.common import LOG, keyisset +from ecs_composex.common.aws import define_tagsgroups_filter_tags +from ecs_composex.s3.s3_params import S3_ARN_REGEX + + +def lookup_s3_bucket(session, bucket_name=None, tags=None): + """ + + :param boto3.session.Session session: + :param str bucket_name: + :param list tags: + :return: + """ + if bucket_name is not None: + client = session.client("s3") + try: + client.head_bucket(Bucket=bucket_name) + return bucket_name + except client.exceptions.NoSuchBucket: + return None + except ClientError as error: + LOG.error(error) + raise + + elif bucket_name is None and tags: + if not isinstance(tags, list): + raise TypeError("Tags must be a list of key/value dict") + filters = define_tagsgroups_filter_tags(tags) + print(filters) + client = session.client("resourcegroupstaggingapi") + buckets_r = client.get_resources( + ResourceTypeFilters=("s3",), TagFilters=filters + ) + if keyisset("ResourceTagMappingList", buckets_r): + resources = buckets_r["ResourceTagMappingList"] + if len(resources) != 1: + raise LookupError( + "Found more than one bucket with the current tags", + [resource["ResourceARN"] for resource in resources], + "Expected to match only 1 bucket.", + ) + s3_filter = re.compile(S3_ARN_REGEX) + return [ + { + "Name": s3_filter.match(resources[0]["ResourceARN"]).groups()[-1], + "Arn": resources[0]["ResourceARN"], + } + ] + return None diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py index 34d1330b7..428d58ef5 100644 --- a/ecs_composex/s3/s3_ecs.py +++ b/ecs_composex/s3/s3_ecs.py @@ -19,12 +19,71 @@ Functions to pass permissions to Services to access S3 buckets. """ +from troposphere.s3 import Bucket -def s3_to_ecs(queues, services_stack, services_families, res_root_stack, settings, **kwargs): +from ecs_composex.common import LOG, NONALPHANUM +from ecs_composex.common.stacks import ComposeXStack +from ecs_composex.resource_permissions import apply_iam_based_resources +from ecs_composex.resource_settings import ( + generate_resource_envvars, + generate_resource_permissions, + validate_lookup_resource, +) +from ecs_composex.s3.s3_aws import lookup_s3_bucket +from ecs_composex.s3.s3_params import S3_BUCKET_NAME +from ecs_composex.s3.s3_perms import generate_s3_permissions + + +def handle_new_buckets( + xresources, + services_families, + services_stack, + res_root_stack, + l_buckets, + nested=False, +): + buckets_r = [] + s_resources = res_root_stack.stack_template.resources + for resource_name in s_resources: + if isinstance(s_resources[resource_name], Bucket): + buckets_r.append(s_resources[resource_name].title) + elif issubclass(type(s_resources[resource_name]), ComposeXStack): + handle_new_buckets( + xresources, + services_families, + services_stack, + s_resources[resource_name], + l_buckets, + nested=True, + ) + + for bucket_name in xresources: + if bucket_name in buckets_r or NONALPHANUM.sub("", bucket_name) in buckets_r: + perms = generate_s3_permissions( + NONALPHANUM.sub("", bucket_name), S3_BUCKET_NAME + ) + envvars = generate_resource_envvars( + bucket_name, xresources[bucket_name], S3_BUCKET_NAME + ) + apply_iam_based_resources( + xresources[bucket_name], + services_families, + services_stack, + res_root_stack, + envvars, + perms, + nested, + ) + del l_buckets[bucket_name] + + +def s3_to_ecs( + xresources, services_stack, services_families, res_root_stack, settings, **kwargs +): """ Function to handle permissions assignment to ECS services. - :param queues: x-sqs queues defined in compose file + :param xresources: x-sqs queues defined in compose file :param ecs_composex.common.stack.ComposeXStack services_stack: services root stack :param services_families: services families :param ecs_composex.common.stack.ComposeXStack res_root_stack: s3 root stack @@ -32,4 +91,40 @@ def s3_to_ecs(queues, services_stack, services_families, res_root_stack, setting :param dict kwargs: :return: """ - + l_buckets = xresources.copy() + handle_new_buckets( + xresources, services_families, services_stack, res_root_stack, l_buckets + ) + for bucket_name in l_buckets: + bucket = xresources[bucket_name] + bucket_res_name = NONALPHANUM.sub("", bucket_name) + validate_lookup_resource(bucket_res_name, bucket, res_root_stack) + found_resources = lookup_s3_bucket( + settings.session, tags=bucket["Lookup"]["Tags"] + ) + if not found_resources: + LOG.warning( + f"404 not buckets found with the provided tags was found in definition {bucket_name}." + ) + continue + for found_bucket in found_resources: + bucket.update(found_bucket) + perms = generate_s3_permissions( + found_bucket["Name"], + S3_BUCKET_NAME, + arn=found_bucket["Arn"], + ) + envvars = generate_resource_envvars( + bucket_name, + xresources[bucket_name], + S3_BUCKET_NAME, + arn=found_bucket["Name"], + ) + apply_iam_based_resources( + bucket, + services_families, + services_stack, + res_root_stack, + envvars, + perms, + ) diff --git a/ecs_composex/s3/s3_params.py b/ecs_composex/s3/s3_params.py index aff3d4b93..ab4f73e74 100644 --- a/ecs_composex/s3/s3_params.py +++ b/ecs_composex/s3/s3_params.py @@ -21,6 +21,10 @@ RES_KEY = f"{X_KEY}{path.basename(path.dirname(path.abspath(__file__)))}" +S3_ARN_REGEX = r"arn:(aws|aws-gov|aws-cn):s3:::([a-zA-Z0-9-.]+$)" + +S3_BUCKET_ARN_T = "BucketArn" +S3_BUCKET_ARN = Parameter(S3_BUCKET_ARN_T, Type="String", AllowedPattern=S3_ARN_REGEX) S3_BUCKET_NAME_T = "BucketName" S3_BUCKET_NAME = Parameter( S3_BUCKET_NAME_T, Type="String", AllowedPattern=r"^[a-z0-9-.]+$" diff --git a/ecs_composex/s3/s3_perms.py b/ecs_composex/s3/s3_perms.py new file mode 100644 index 000000000..f188eac9d --- /dev/null +++ b/ecs_composex/s3/s3_perms.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# ECS ComposeX +# Copyright (C) 2020 John Mille +# # +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from os import path +from json import loads + +from troposphere import Sub, ImportValue, Parameter +from troposphere.iam import Policy as IamPolicy + +from ecs_composex.common import LOG, NONALPHANUM +from ecs_composex.common.ecs_composex import CFN_EXPORT_DELIMITER as DELIM +from ecs_composex.common.cfn_params import ROOT_STACK_NAME +from ecs_composex.resource_settings import generate_export_strings + + +def get_access_types(): + with open( + f"{path.abspath(path.dirname(__file__))}/s3_perms.json", + "r", + encoding="utf-8-sig", + ) as perms_fd: + return loads(perms_fd.read()) + + +def generate_s3_bucket_resource_strings(res_name, attribute): + """ + Function to generate the SSM and CFN import/export strings + Returns the import in a tuple + + :param str res_name: name of the queue as defined in ComposeX File + :param str|Parameter attribute: The attribute to use in Import Name. + + :returns: ImportValue for CFN + :rtype: ImportValue + """ + if isinstance(attribute, str): + bucket_string = ( + f"${{{ROOT_STACK_NAME.title}}}{DELIM}{res_name}{DELIM}{attribute}" + ) + objects_string = ( + f"${{{ROOT_STACK_NAME.title}}}{DELIM}{res_name}{DELIM}{attribute}/*" + ) + elif isinstance(attribute, Parameter): + bucket_string = ( + f"${{{ROOT_STACK_NAME.title}}}{DELIM}{res_name}{DELIM}{attribute.title}" + ) + objects_string = ( + f"${{{ROOT_STACK_NAME.title}}}{DELIM}{res_name}{DELIM}{attribute.title}/*" + ) + else: + raise TypeError("Attribute can only be a string or Parameter") + + return [ImportValue(Sub(bucket_string)), ImportValue(Sub(objects_string))] + + +def generate_s3_permissions(resource_name, attribute, arn=None): + """ + Function to generate IAM permissions for a given x-resource. Returns the mapping of these for the given resource. + + :param str resource_name: The name of the resource + :param str attribute: the attribute of the resource we are using for Import + :param str arn: The ARN of the resource if already looked up. + :return: dict of the IAM policies associated with the resource. + :rtype dict: + """ + resource_policies = {} + policies = get_access_types() + for a_type in policies: + clean_policy = {"Version": "2012-10-17", "Statement": []} + LOG.debug(a_type) + policy_doc = policies[a_type].copy() + policy_doc["Sid"] = Sub(NONALPHANUM.sub("", f"{a_type}To{resource_name}")) + policy_doc["Resource"] = ( + generate_export_strings(resource_name, attribute) + if not arn + else [f"{arn}/*", arn] + ) + clean_policy["Statement"].append(policy_doc) + resource_policies[a_type] = IamPolicy( + PolicyName=Sub( + NONALPHANUM.sub( + "", f"{a_type}{resource_name}${{{ROOT_STACK_NAME.title}}}" + ) + ), + PolicyDocument=clean_policy, + ) + return resource_policies diff --git a/ecs_composex/s3/s3_template.py b/ecs_composex/s3/s3_template.py index ac6f6b044..3dd3981be 100644 --- a/ecs_composex/s3/s3_template.py +++ b/ecs_composex/s3/s3_template.py @@ -82,12 +82,16 @@ def define_accelerate_config(properties, settings): :param settings: :return: """ - config = s3.AccelerateConfiguration(AccelerationStatus="SUSPENDED") + config = s3.AccelerateConfiguration( + AccelerationStatus=s3.s3_transfer_acceleration_status("Suspended") + ) if keyisset("AccelerateConfiguration", properties): config = s3.AccelerateConfiguration( - AccelerationStatus="SUSPENDED" + AccelerationStatus=s3.s3_transfer_acceleration_status("Suspended") if not keyisset("AccelerationStatus", properties["AccelerateConfiguration"]) - else properties["AccelerateConfiguration"]["AccelerationStatus"] + else s3.s3_transfer_acceleration_status( + properties["AccelerateConfiguration"]["AccelerationStatus"] + ) ) elif keyisset("AccelerationStatus", settings): config = s3.AccelerateConfiguration( @@ -113,7 +117,9 @@ def define_bucket(bucket_name, res_name, definition): if not keyisset("AccessControl", properties) else properties["AccessControl"], "BucketEncryption": handle_bucket_encryption(definition, settings), - "BucketName": bucket_name, + "BucketName": definition["BucketName"] + if keyisset("BucketName", definition) + else Ref(AWS_NO_VALUE), "ObjectLockEnabled": False if not keyisset("ObjectLockEnabled", properties) else properties["ObjectLockEnabled"], @@ -132,7 +138,6 @@ def define_bucket(bucket_name, res_name, definition): ) if keyisset("VersioningConfiguration", properties) else Ref(AWS_NO_VALUE), - "WebsiteConfiguration": s3.WebsiteConfiguration, "Metadata": metadata, } bucket = s3.Bucket(res_name, **props) diff --git a/use-cases/s3/lookup_use_create_buckets.yml b/use-cases/s3/lookup_use_create_buckets.yml new file mode 100644 index 000000000..ee8a7cf3e --- /dev/null +++ b/use-cases/s3/lookup_use_create_buckets.yml @@ -0,0 +1,19 @@ +version: "3.8" + +x-s3: + bucket-01: + Properties: + BucketName: bucket-01 + Settings: {} + Services: + - name: app03 + access: RWObjects + + bucket-02: + Lookup: + Tags: + - aws:cloudformation:logical-id: ArtifactsBucket + - aws:cloudformation:stack-name: pipeline-shared-buckets + Services: + - name: app03 + access: RWObjects diff --git a/use-cases/s3/simple_s3_bucket.yml b/use-cases/s3/simple_s3_bucket.yml new file mode 100644 index 000000000..7d2f91a79 --- /dev/null +++ b/use-cases/s3/simple_s3_bucket.yml @@ -0,0 +1,9 @@ +version: "3.8" + +x-s3: + bucket-01: + Properties: + BucketName: bucket-01 + Services: + - name: app03 + access: RWObjects From 38ff579fe8f9e89204981ec09f61021f5d23400f Mon Sep 17 00:00:00 2001 From: John Preston Date: Fri, 16 Oct 2020 18:20:50 +0100 Subject: [PATCH 03/13] Some refactor to allow for Lookup only resources categories --- ecs_composex/common/aws.py | 53 +++++---------- ecs_composex/common/compose_resources.py | 7 +- ecs_composex/common/stacks.py | 1 + ecs_composex/ecs_composex.py | 61 ++++++++++++----- ecs_composex/rds/rds_aws.py | 10 +-- ecs_composex/s3/s3_aws.py | 83 ++++++++++++------------ ecs_composex/s3/s3_ecs.py | 79 ++++++++++++---------- ecs_composex/s3/s3_stack.py | 25 ++++++- pytests/test_common_aws.py | 10 +-- 9 files changed, 187 insertions(+), 142 deletions(-) diff --git a/ecs_composex/common/aws.py b/ecs_composex/common/aws.py index 4b73c3b98..cd7841113 100644 --- a/ecs_composex/common/aws.py +++ b/ecs_composex/common/aws.py @@ -42,19 +42,20 @@ def define_tagsgroups_filter_tags(tags): return filters -def get_resources_from_tags(session, service_code, res_type, search_tags): +def get_resources_from_tags(session, aws_resource_search, search_tags): """ :param boto3.session.Session session: The boto3 session for API calls - :param str service_code: AWS Service short code, ie. rds, ec2 + :param str aws_resource_search: AWS Service short code, ie. rds, ec2 :param str res_type: Resource type we are after within the AWS Service, ie. cluster, instance :param list search_tags: The tags to search the resource with. :return: """ + LOG.info(aws_resource_search) try: client = session.client("resourcegroupstaggingapi") resources_r = client.get_resources( - ResourceTypeFilters=[f"{service_code}:{res_type}"], TagFilters=search_tags + ResourceTypeFilters=[aws_resource_search], TagFilters=search_tags ) return resources_r except ClientError as error: @@ -100,15 +101,14 @@ def handle_multi_results(arns, name, res_type, regexp): ) -def handle_search_results(arns, name, res_types, res_type, service_code): +def handle_search_results(arns, name, res_types, aws_resource_search): """ Function to parse tag resource search results :param list arns: :param str name: :param dict res_types: - :param str res_type: - :param str service_code: + :param str aws_resource_search: :return: """ if not arns: @@ -116,13 +116,15 @@ def handle_search_results(arns, name, res_types, res_type, service_code): "No resources were found with the provided tags and information" ) if arns and isinstance(name, str): - return handle_multi_results(arns, name, res_type, res_types[res_type]["regexp"]) + return handle_multi_results( + arns, name, aws_resource_search, res_types[aws_resource_search]["regexp"] + ) elif not name and len(arns) == 1: - LOG.info(f"Matched {service_code}:{res_type}") + LOG.info(f"Matched {aws_resource_search} to AWS Resource") return arns[0] elif not name and len(arns) != 1: raise LookupError( - f"More than one {service_code}:{res_type} was found with the current tags." + f"More than one resource {name}:{aws_resource_search} was found with the current tags." "Found", arns, ) @@ -146,51 +148,32 @@ def validate_search_input(res_types, res_type): ) -def find_aws_resource_arn_from_tags_api( - info, session, service_code, res_type, types=None -): +def find_aws_resource_arn_from_tags_api(info, session, aws_resource_search, types=None): """ Function to find the RDS DB based on info :param dict info: :param boto3.session.Session session: Boto3 session for clients - :param str res_type: Resource type we are after within the AWS Service, ie. cluster, instance - :param str service_code: AWS Service short code, ie. rds, ec2 + :param str aws_resource_search: Resource type we are after within the AWS Service, ie. cluster, instance :param dict types: Additional types to match. :return: """ res_types = { - "secret": { + "secretsmanager:secret": { "regexp": r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]+)$" }, } if types is not None and isinstance(types, dict): res_types.update(types) - validate_search_input(res_types, res_type) + validate_search_input(res_types, aws_resource_search) search_tags = ( define_tagsgroups_filter_tags(info["Tags"]) if keyisset("Tags", info) else () ) name = info["Name"] if keyisset("Name", info) else None - resources_r = get_resources_from_tags(session, service_code, res_type, search_tags) - arns = [i["ResourceARN"] for i in resources_r["ResourceTagMappingList"]] - return handle_search_results(arns, name, res_types, res_type, service_code) - -def define_tagsgroups_filter_tags(tags): - """ - Function to create the filters out of tags list - - :param list tags: list of Key/Value dict - :return: filters - :rtype: list - """ - filters = [] - for tag in tags: - key = list(tag.keys())[0] - filter_name = key - filter_value = tag[key] - filters.append({"Key": filter_name, "Values": (filter_value,)}) - return filters + resources_r = get_resources_from_tags(session, aws_resource_search, search_tags) + arns = [i["ResourceARN"] for i in resources_r["ResourceTagMappingList"]] + return handle_search_results(arns, name, res_types, aws_resource_search) def get_region_azs(session): diff --git a/ecs_composex/common/compose_resources.py b/ecs_composex/common/compose_resources.py index e976eff1b..87b65dc97 100644 --- a/ecs_composex/common/compose_resources.py +++ b/ecs_composex/common/compose_resources.py @@ -22,7 +22,7 @@ from troposphere import Sub from troposphere.ecs import Environment -from ecs_composex.common import NONALPHANUM, keyisset +from ecs_composex.common import LOG, NONALPHANUM, keyisset from ecs_composex.resource_settings import generate_export_strings from ecs_composex.common.cfn_params import ROOT_STACK_NAME @@ -38,9 +38,12 @@ def set_resources(settings, resource_class, res_key): if not keyisset(res_key, settings.compose_content): return for resource_name in settings.compose_content[res_key]: - settings.compose_content[res_key][resource_name] = resource_class( + new_definition = resource_class( resource_name, settings.compose_content[res_key][resource_name] ) + LOG.debug(type(new_definition)) + LOG.debug(new_definition.__dict__) + settings.compose_content[res_key][resource_name] = new_definition class Service(object): diff --git a/ecs_composex/common/stacks.py b/ecs_composex/common/stacks.py index 8e019f82a..f19b7db68 100644 --- a/ecs_composex/common/stacks.py +++ b/ecs_composex/common/stacks.py @@ -69,6 +69,7 @@ class ComposeXStack(Stack, object): "UpdatePolicy", "UpdateReplacePolicy", ] + is_void = False def __init__( self, title, stack_template, stack_parameters=None, file_name=None, **kwargs diff --git a/ecs_composex/ecs_composex.py b/ecs_composex/ecs_composex.py index 77a0b968d..8b7813ff0 100644 --- a/ecs_composex/ecs_composex.py +++ b/ecs_composex/ecs_composex.py @@ -144,6 +144,29 @@ def get_mod_class(module_name): return the_class +def invoke_x_to_ecs(module, settings, services_stack, services_families, resource): + """ + + :param str module: + :param ecs_composex.common.settings.ComposeXSettings settings: The compose file content + :param ecs_composex.ecs.ServicesStack services_stack: root stack for services. + :param dict services_families: Families and services mappings + :param resource: The XStack resource + :return: + """ + composex_key = f"{X_KEY}{module}" + ecs_function = get_mod_function(f"{module}.{module}_ecs", f"{module}_to_ecs") + if ecs_function: + LOG.debug(ecs_function) + ecs_function( + settings.compose_content[composex_key], + services_stack, + services_families, + resource, + settings, + ) + + def apply_x_configs_to_ecs( settings, root_template, services_stack, services_families, **kwargs ): @@ -165,19 +188,9 @@ def apply_x_configs_to_ecs( and resource_name in SUPPORTED_X_MODULES ): module = getattr(resource, "title") - composex_key = f"{X_KEY}{module}" - ecs_function = get_mod_function( - f"{module}.{module}_ecs", f"{module}_to_ecs" + invoke_x_to_ecs( + module, settings, services_stack, services_families, resource ) - if ecs_function: - LOG.debug(ecs_function) - ecs_function( - settings.compose_content[composex_key], - services_stack, - services_families, - resource, - settings, - ) def apply_x_to_x_configs(root_template, settings): @@ -229,7 +242,9 @@ def add_compute(root_template, settings, vpc_stack): return root_template.add_resource(compute_stack) -def add_x_resources(root_template, settings, vpc_stack=None): +def add_x_resources( + root_template, settings, services_stack, services_families, vpc_stack=None +): """ Function to add each X resource from the compose file """ @@ -252,7 +267,17 @@ def add_x_resources(root_template, settings, vpc_stack=None): xstack.get_from_vpc_stack(vpc_stack) elif not vpc_stack and key in tcp_services: xstack.no_vpc_parameters() - root_template.add_resource(xstack) + LOG.debug(xstack, xstack.is_void) + if xstack.is_void: + invoke_x_to_ecs( + res_type, settings, services_stack, services_families, xstack + ) + elif ( + hasattr(xstack, "title") + and hasattr(xstack, "stack_template") + and not xclass.is_void + ): + root_template.add_resource(xstack) def create_services(root_stack, settings, vpc_stack, dns_params, create_cluster): @@ -333,7 +358,13 @@ def generate_full_template(settings): services_stack = create_services( root_stack, settings, vpc_stack, dns_settings.nested_params, create_cluster ) - add_x_resources(root_stack.stack_template, settings, vpc_stack=vpc_stack) + add_x_resources( + root_stack.stack_template, + settings, + services_stack, + services_families, + vpc_stack=vpc_stack, + ) apply_x_configs_to_ecs( settings, root_stack.stack_template, services_stack, services_families ) diff --git a/ecs_composex/rds/rds_aws.py b/ecs_composex/rds/rds_aws.py index c957391e8..9733453fb 100644 --- a/ecs_composex/rds/rds_aws.py +++ b/ecs_composex/rds/rds_aws.py @@ -128,7 +128,7 @@ def handle_secret(lookup, db_config, session): """ if keyisset("secret", lookup): secret_arn = find_aws_resource_arn_from_tags_api( - lookup["secret"], session, "secretsmanager", "secret" + lookup["secret"], session, "secretsmanager:secret" ) if secret_arn and db_config: db_config.update({"SecretArn": secret_arn}) @@ -159,8 +159,10 @@ def lookup_rds_resource(lookup, session): :return: """ rds_types = { - "db": {"regexp": r"(?:^arn:aws(?:-[a-z]+)?:rds:[\w-]+:[0-9]{12}:db:)([\S]+)$"}, - "cluster": { + "rds:db": { + "regexp": r"(?:^arn:aws(?:-[a-z]+)?:rds:[\w-]+:[0-9]{12}:db:)([\S]+)$" + }, + "rds:cluster": { "regexp": r"(?:^arn:aws(?:-[a-z]+)?:rds:[\w-]+:[0-9]{12}:cluster:)([\S]+)$" }, } @@ -170,7 +172,7 @@ def lookup_rds_resource(lookup, session): elif keyisset("db", lookup): res_type = "db" db_arn = find_aws_resource_arn_from_tags_api( - lookup[res_type], session, "rds", res_type, types=rds_types + lookup[res_type], session, f"rds:{res_type}", types=rds_types ) if not db_arn: return None diff --git a/ecs_composex/s3/s3_aws.py b/ecs_composex/s3/s3_aws.py index 1ce6a2d87..5f82bab2e 100644 --- a/ecs_composex/s3/s3_aws.py +++ b/ecs_composex/s3/s3_aws.py @@ -26,49 +26,52 @@ from ecs_composex.common import LOG, keyisset from ecs_composex.common.aws import define_tagsgroups_filter_tags from ecs_composex.s3.s3_params import S3_ARN_REGEX +from ecs_composex.common.aws import find_aws_resource_arn_from_tags_api -def lookup_s3_bucket(session, bucket_name=None, tags=None): +# def return_db_config(bucket_arn, session): +# """ +# +# :param str bucket_arn: +# :param boto3.session.Session session: +# :return: +# """ +# client = session.client("s3") +# try: +# client.head_bucket(Bucket=bucket_name) +# return bucket_name +# except client.exceptions.NoSuchBucket: +# return None +# except ClientError as error: +# LOG.error(error) +# raise + + +def lookup_bucket(lookup, session): """ + Function to find the DB in AWS account - :param boto3.session.Session session: - :param str bucket_name: - :param list tags: + :param dict lookup: The Lookup definition for DB + :param boto3.session.Session session: Boto3 session for clients :return: """ - if bucket_name is not None: - client = session.client("s3") - try: - client.head_bucket(Bucket=bucket_name) - return bucket_name - except client.exceptions.NoSuchBucket: - return None - except ClientError as error: - LOG.error(error) - raise - - elif bucket_name is None and tags: - if not isinstance(tags, list): - raise TypeError("Tags must be a list of key/value dict") - filters = define_tagsgroups_filter_tags(tags) - print(filters) - client = session.client("resourcegroupstaggingapi") - buckets_r = client.get_resources( - ResourceTypeFilters=("s3",), TagFilters=filters - ) - if keyisset("ResourceTagMappingList", buckets_r): - resources = buckets_r["ResourceTagMappingList"] - if len(resources) != 1: - raise LookupError( - "Found more than one bucket with the current tags", - [resource["ResourceARN"] for resource in resources], - "Expected to match only 1 bucket.", - ) - s3_filter = re.compile(S3_ARN_REGEX) - return [ - { - "Name": s3_filter.match(resources[0]["ResourceARN"]).groups()[-1], - "Arn": resources[0]["ResourceARN"], - } - ] - return None + rds_types = { + "s3": {"regexp": r"(?:^arn:aws(?:-[a-z]+)?:s3:::)([\S]+)$"}, + } + res_type = None + if keyisset("bucket", lookup): + res_type = "bucket" + bucket_arn = find_aws_resource_arn_from_tags_api( + lookup[res_type], + session, + "rds", + res_type, + types=rds_types, + service_is_type=True, + ) + print(bucket_arn) + if not bucket_arn: + return None + # bucket_config = return_db_config(bucket_arn, session) + # LOG.debug(bucket_config) + # return bucket_config diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py index 428d58ef5..e166cd12f 100644 --- a/ecs_composex/s3/s3_ecs.py +++ b/ecs_composex/s3/s3_ecs.py @@ -29,7 +29,6 @@ generate_resource_permissions, validate_lookup_resource, ) -from ecs_composex.s3.s3_aws import lookup_s3_bucket from ecs_composex.s3.s3_params import S3_BUCKET_NAME from ecs_composex.s3.s3_perms import generate_s3_permissions @@ -43,6 +42,8 @@ def handle_new_buckets( nested=False, ): buckets_r = [] + if res_root_stack.is_void: + return s_resources = res_root_stack.stack_template.resources for resource_name in s_resources: if isinstance(s_resources[resource_name], Bucket): @@ -92,39 +93,45 @@ def s3_to_ecs( :return: """ l_buckets = xresources.copy() - handle_new_buckets( - xresources, services_families, services_stack, res_root_stack, l_buckets - ) - for bucket_name in l_buckets: - bucket = xresources[bucket_name] - bucket_res_name = NONALPHANUM.sub("", bucket_name) - validate_lookup_resource(bucket_res_name, bucket, res_root_stack) - found_resources = lookup_s3_bucket( - settings.session, tags=bucket["Lookup"]["Tags"] - ) - if not found_resources: - LOG.warning( - f"404 not buckets found with the provided tags was found in definition {bucket_name}." - ) - continue - for found_bucket in found_resources: - bucket.update(found_bucket) - perms = generate_s3_permissions( - found_bucket["Name"], - S3_BUCKET_NAME, - arn=found_bucket["Arn"], - ) - envvars = generate_resource_envvars( - bucket_name, - xresources[bucket_name], - S3_BUCKET_NAME, - arn=found_bucket["Name"], - ) - apply_iam_based_resources( - bucket, - services_families, - services_stack, - res_root_stack, - envvars, - perms, + for res_name in xresources: + res = xresources[res_name] + if res.properties and not res.lookup: + print(f"New resource to create {res.logical_name}") + handle_new_buckets( + xresources, services_families, services_stack, res_root_stack, l_buckets ) + elif res.lookup: + print(f"Resource to find, {res.logical_name}") + # handle_new_buckets( + # xresources, services_families, services_stack, res_root_stack, l_buckets + # ) + # for bucket_name in l_buckets: + # bucket = xresources[bucket_name] + # bucket_res_name = NONALPHANUM.sub("", bucket_name) + # validate_lookup_resource(bucket_res_name, bucket, res_root_stack) + # if not found_resources: + # LOG.warning( + # f"404 not buckets found with the provided tags was found in definition {bucket_name}." + # ) + # continue + # for found_bucket in found_resources: + # bucket.update(found_bucket) + # perms = generate_s3_permissions( + # found_bucket["Name"], + # S3_BUCKET_NAME, + # arn=found_bucket["Arn"], + # ) + # envvars = generate_resource_envvars( + # bucket_name, + # xresources[bucket_name], + # S3_BUCKET_NAME, + # arn=found_bucket["Name"], + # ) + # apply_iam_based_resources( + # bucket, + # services_families, + # services_stack, + # res_root_stack, + # envvars, + # perms, + # ) diff --git a/ecs_composex/s3/s3_stack.py b/ecs_composex/s3/s3_stack.py index cf3d484c8..be7886a7b 100644 --- a/ecs_composex/s3/s3_stack.py +++ b/ecs_composex/s3/s3_stack.py @@ -20,11 +20,12 @@ """ from troposphere import Ref, GetAtt -from troposphere.s3 import Bucket +from troposphere.s3 import Bucket as S3Bucket from ecs_composex.common import LOG, keyisset, build_template, NONALPHANUM from ecs_composex.common.stacks import ComposeXStack from ecs_composex.common.outputs import ComposeXOutput +from ecs_composex.common.compose_resources import XResource, set_resources from ecs_composex.s3.s3_params import RES_KEY, S3_BUCKET_NAME_T from ecs_composex.s3.s3_template import generate_bucket @@ -43,6 +44,14 @@ def create_s3_template(settings): if not keyisset(RES_KEY, settings.compose_content): return None buckets = settings.compose_content[RES_KEY] + if not [ + buckets[bucket_name] + for bucket_name in buckets + if buckets[bucket_name].properties + ]: + LOG.info("There are no buckets to create.") + return None + if len(list(buckets.keys())) <= CFN_MAX_OUTPUTS: mono_template = True @@ -74,11 +83,21 @@ def create_s3_template(settings): return template -class XResource(ComposeXStack): +class Bucket(XResource): + """ + Class for S3 bucket. + """ + + +class XStack(ComposeXStack): """ Class to handle S3 buckets """ def __init__(self, title, settings, **kwargs): + set_resources(settings, Bucket, RES_KEY) stack_template = create_s3_template(settings) - super().__init__(title, stack_template, **kwargs) + if stack_template: + super().__init__(title, stack_template, **kwargs) + else: + self.is_void = True diff --git a/pytests/test_common_aws.py b/pytests/test_common_aws.py index de34d27c7..ce527cc18 100644 --- a/pytests/test_common_aws.py +++ b/pytests/test_common_aws.py @@ -59,16 +59,12 @@ def test_multi_arns_exceptions(multi_matching_arns): def test_handle_results_exceptions(): - handle_search_results(["arn:aws:s3:::sombucket-found"], None, {}, "bucket", "s3") + handle_search_results(["arn:aws:s3:::sombucket-found"], None, {}, "s3") with raises(LookupError): - handle_search_results([], None, {}, "bucket", "s3") + handle_search_results([], None, {}, "s3") with raises(LookupError): handle_search_results( - ["arn:aws:s3:::bucket1", "arn:aws:s3:::anotherone"], - None, - {}, - "bucket", - "s3", + ["arn:aws:s3:::bucket1", "arn:aws:s3:::anotherone"], None, {}, "s3" ) From 8a238503224de9152219b8b32aa0dd346cb1ab93 Mon Sep 17 00:00:00 2001 From: John Preston Date: Sat, 17 Oct 2020 07:58:02 +0100 Subject: [PATCH 04/13] Good WIP for S3 lookup --- ecs_composex/common/aws.py | 7 +- ecs_composex/resource_settings.py | 4 +- ecs_composex/s3/s3_aws.py | 74 ++++--- ecs_composex/s3/s3_ecs.py | 227 +++++++++++++++++---- ecs_composex/s3/s3_perms.json | 93 ++++++--- ecs_composex/s3/s3_perms.py | 3 + use-cases/s3/lookup_use_create_buckets.yml | 8 +- 7 files changed, 310 insertions(+), 106 deletions(-) diff --git a/ecs_composex/common/aws.py b/ecs_composex/common/aws.py index cd7841113..94df94154 100644 --- a/ecs_composex/common/aws.py +++ b/ecs_composex/common/aws.py @@ -47,11 +47,9 @@ def get_resources_from_tags(session, aws_resource_search, search_tags): :param boto3.session.Session session: The boto3 session for API calls :param str aws_resource_search: AWS Service short code, ie. rds, ec2 - :param str res_type: Resource type we are after within the AWS Service, ie. cluster, instance :param list search_tags: The tags to search the resource with. :return: """ - LOG.info(aws_resource_search) try: client = session.client("resourcegroupstaggingapi") resources_r = client.get_resources( @@ -134,9 +132,8 @@ def validate_search_input(res_types, res_type): """ Function to validate the search query - :param info: - :param res_types: - :param res_type: + :param dict res_types: + :param str res_type: :return: """ diff --git a/ecs_composex/resource_settings.py b/ecs_composex/resource_settings.py index 024284809..58964a294 100644 --- a/ecs_composex/resource_settings.py +++ b/ecs_composex/resource_settings.py @@ -56,9 +56,9 @@ def generate_resource_permissions(resource_name, policies, attribute, arn=None): Function to generate IAM permissions for a given x-resource. Returns the mapping of these for the given resource. :param str resource_name: The name of the resource - :param str attribute: the attribute of the resource we are using for Import + :param str,None attribute: the attribute of the resource we are using for Import :param dict policies: the policies associated with the x-resource type. - :param str arn: The ARN of the resource if already looked up. + :param str,AWSHelper arn: The ARN of the resource if already looked up. :return: dict of the IAM policies associated with the resource. :rtype dict: """ diff --git a/ecs_composex/s3/s3_aws.py b/ecs_composex/s3/s3_aws.py index 5f82bab2e..1327699cc 100644 --- a/ecs_composex/s3/s3_aws.py +++ b/ecs_composex/s3/s3_aws.py @@ -29,25 +29,45 @@ from ecs_composex.common.aws import find_aws_resource_arn_from_tags_api -# def return_db_config(bucket_arn, session): -# """ -# -# :param str bucket_arn: -# :param boto3.session.Session session: -# :return: -# """ -# client = session.client("s3") -# try: -# client.head_bucket(Bucket=bucket_name) -# return bucket_name -# except client.exceptions.NoSuchBucket: -# return None -# except ClientError as error: -# LOG.error(error) -# raise +def return_bucket_config(bucket_arn, session): + """ + + :param str bucket_arn: + :param boto3.session.Session session: + :return: + """ + bucket_name_finder = re.compile(r"([a-zA-Z0-9.\-_]{1,255}$)") + bucket_name = bucket_name_finder.findall(bucket_arn)[-1] + bucket_config = {"Name": bucket_name, "Arn": bucket_arn} + client = session.client("s3") + try: + client.head_bucket(Bucket=bucket_name) + try: + encryption_config_r = client.get_bucket_encryption(Bucket=bucket_name) + if keyisset("ServerSideEncryptionConfiguration", encryption_config_r): + bucket_config.update( + { + "ServerSideEncryptionConfiguration": encryption_config_r[ + "ServerSideEncryptionConfiguration" + ] + } + ) + except ClientError as error: + if ( + not error.response["Error"]["Code"] + == "ServerSideEncryptionConfigurationNotFoundError" + ): + raise + LOG.error(error.response["Error"]["Message"]) + return bucket_config + except client.exceptions.NoSuchBucket: + return None + except ClientError as error: + LOG.error(error) + raise -def lookup_bucket(lookup, session): +def lookup_bucket_config(lookup, session): """ Function to find the DB in AWS account @@ -55,23 +75,17 @@ def lookup_bucket(lookup, session): :param boto3.session.Session session: Boto3 session for clients :return: """ - rds_types = { + s3_types = { "s3": {"regexp": r"(?:^arn:aws(?:-[a-z]+)?:s3:::)([\S]+)$"}, } - res_type = None - if keyisset("bucket", lookup): - res_type = "bucket" bucket_arn = find_aws_resource_arn_from_tags_api( - lookup[res_type], + lookup, session, - "rds", - res_type, - types=rds_types, - service_is_type=True, + "s3", + types=s3_types, ) - print(bucket_arn) if not bucket_arn: return None - # bucket_config = return_db_config(bucket_arn, session) - # LOG.debug(bucket_config) - # return bucket_config + config = return_bucket_config(bucket_arn, session) + LOG.debug(config) + return config diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py index e166cd12f..48783eca2 100644 --- a/ecs_composex/s3/s3_ecs.py +++ b/ecs_composex/s3/s3_ecs.py @@ -19,18 +19,26 @@ Functions to pass permissions to Services to access S3 buckets. """ +from json import dumps +from troposphere import FindInMap, Sub from troposphere.s3 import Bucket -from ecs_composex.common import LOG, NONALPHANUM +from ecs_composex.common import LOG, NONALPHANUM, keyisset from ecs_composex.common.stacks import ComposeXStack -from ecs_composex.resource_permissions import apply_iam_based_resources +from ecs_composex.resource_permissions import ( + apply_iam_based_resources, + add_iam_policy_to_service_task_role, +) from ecs_composex.resource_settings import ( generate_resource_envvars, generate_resource_permissions, validate_lookup_resource, ) +from ecs_composex.ecs.ecs_template import get_service_family_name from ecs_composex.s3.s3_params import S3_BUCKET_NAME -from ecs_composex.s3.s3_perms import generate_s3_permissions +from ecs_composex.s3.s3_perms import ACCESS_TYPES, generate_s3_permissions +from ecs_composex.s3.s3_aws import lookup_bucket_config +from ecs_composex.kms.kms_perms import ACCESS_TYPES as KMS_ACCESS_TYPES def handle_new_buckets( @@ -78,6 +86,158 @@ def handle_new_buckets( del l_buckets[bucket_name] +def get_bucket_kms_key_from_config(bucket_config): + """ + Functiont to get the KMS Encryption key if defined. + + :param bucket_config: + :return: + """ + rules = ( + [] + if not ( + keyisset("ServerSideEncryptionConfiguration", bucket_config) + and keyisset("Rules", bucket_config["ServerSideEncryptionConfiguration"]) + ) + else bucket_config["ServerSideEncryptionConfiguration"]["Rules"] + ) + for rule in rules: + if keyisset("ApplyServerSideEncryptionByDefault", rule): + settings = rule["ApplyServerSideEncryptionByDefault"] + if ( + keyisset("SSEAlgorithm", settings) + and settings["SSEAlgorithm"] == "aws:kms" + and keyisset("KMSMasterKeyID", settings) + ): + return settings["KMSMasterKeyID"] + return None + + +def define_bucket_mappings(buckets_mappings, buckets, settings): + """ + Function to populate bucket mapping + + :param buckets_mappings: + :return: + """ + for bucket in buckets: + bucket_config = lookup_bucket_config(bucket.lookup, settings.session) + buckets_mappings.update( + { + bucket.logical_name: { + "Name": bucket_config["Name"], + "Arn": bucket_config["Arn"], + } + } + ) + bucket_key = get_bucket_kms_key_from_config(bucket_config) + if bucket_key: + LOG.info(f"Identified CMK {bucket_key} to be default key for encryption") + buckets_mappings[bucket.logical_name]["KmsKey"] = bucket_key + else: + LOG.info( + "No KMS Key has been identified to encrypt the bucket. Won't grant service access." + ) + + +def define_bucket_access(bucket, access, service_template, service_family, family_wide): + """ + Function to create the IAM policy for the service access to bucket + + :param bucket: + :param access: + :return: + """ + bucket_key = "bucket" + objects_key = "objects" + bucket_perms = None + objects_perms = None + if isinstance(access, str): + LOG.warn( + "For s3 buckets, you should define a dict for access, with bucket and/or object policies separate." + " Using default RW Objects and ListBucket" + ) + access = {objects_key: "RW", bucket_key: "ListOnly"} + elif isinstance(access, dict): + if not keyisset(objects_key, access) or not keyisset(bucket_key, access): + raise KeyError("You must define at least bucket or object access") + if keyisset(bucket_key, access): + bucket_perms = generate_resource_permissions( + f"BucketAccess{bucket.logical_name}", + ACCESS_TYPES[bucket_key], + None, + arn=FindInMap("s3", bucket.logical_name, "Arn"), + ) + add_iam_policy_to_service_task_role( + service_template, + bucket, + bucket_perms, + access[bucket_key], + service_family, + family_wide, + ) + if keyisset(objects_key, access): + objects_perms = generate_resource_permissions( + f"ObjectsAccess{bucket.logical_name}", + ACCESS_TYPES[objects_key], + None, + arn=Sub( + "${BucketArn}/*", BucketArn=FindInMap("s3", bucket.logical_name, "Arn") + ), + ) + add_iam_policy_to_service_task_role( + service_template, + bucket, + objects_perms, + access[objects_key], + service_family, + family_wide, + ) + + +def assign_lookup_buckets(bucket, mappings, service, services_stack, services_families): + """ + Function to add the lookup bucket to service access + + :param ecs_composex.s3.s3_stacks.Bucket bucket: + :param dict mappings: + :param dict service: + :param ecs_composex.common.stacks.ComposeXStack services_stack: + :param dict services_families: + """ + if not keyisset(bucket.logical_name, mappings): + LOG.warn(f"Bucket {bucket.logical_name} was not found in mappings. Skipping") + return + service_family = get_service_family_name(services_families, service["name"]) + if service_family not in services_stack.stack_template.resources: + raise AttributeError(f"No service {service_family} present in services stack") + family_wide = True if service["name"] in services_families else False + service_stack = services_stack.stack_template.resources[service_family] + service_stack.stack_template.add_mapping("s3", mappings) + service_template = service_stack.stack_template + if keyisset("KmsKey", mappings[bucket.logical_name]): + kms_perms = generate_resource_permissions( + f"{bucket.logical_name}KmsKey", + KMS_ACCESS_TYPES, + None, + arn=FindInMap("s3", bucket.logical_name, "KmsKey"), + ) + add_iam_policy_to_service_task_role( + service_template, + bucket, + kms_perms, + "EncryptDecrypt", + service_family, + family_wide, + ) + if not keyisset("access", service): + LOG.error(f"No access defined for s3 bucket {bucket.name}") + return + define_bucket_access( + bucket, service["access"], service_template, service_family, family_wide + ) + + def s3_to_ecs( xresources, services_stack, services_families, res_root_stack, settings, **kwargs ): @@ -92,46 +252,25 @@ def s3_to_ecs( :param dict kwargs: :return: """ + buckets_mappings = {} l_buckets = xresources.copy() - for res_name in xresources: - res = xresources[res_name] - if res.properties and not res.lookup: - print(f"New resource to create {res.logical_name}") - handle_new_buckets( - xresources, services_families, services_stack, res_root_stack, l_buckets + new_buckets = [ + xresources[name] + for name in xresources + if xresources[name].properties and not xresources[name].lookup + ] + lookup_buckets = [ + xresources[name] for name in xresources if xresources[name].lookup + ] + define_bucket_mappings(buckets_mappings, lookup_buckets, settings) + LOG.debug(dumps(buckets_mappings, indent=4)) + for res in new_buckets: + print(f"New resource to create {res.logical_name}") + handle_new_buckets( + xresources, services_families, services_stack, res_root_stack, l_buckets + ) + for res in lookup_buckets: + for service_def in res.services: + assign_lookup_buckets( + res, buckets_mappings, service_def, services_stack, services_families ) - elif res.lookup: - print(f"Resource to find, {res.logical_name}") - # handle_new_buckets( - # xresources, services_families, services_stack, res_root_stack, l_buckets - # ) - # for bucket_name in l_buckets: - # bucket = xresources[bucket_name] - # bucket_res_name = NONALPHANUM.sub("", bucket_name) - # validate_lookup_resource(bucket_res_name, bucket, res_root_stack) - # if not found_resources: - # LOG.warning( - # f"404 not buckets found with the provided tags was found in definition {bucket_name}." - # ) - # continue - # for found_bucket in found_resources: - # bucket.update(found_bucket) - # perms = generate_s3_permissions( - # found_bucket["Name"], - # S3_BUCKET_NAME, - # arn=found_bucket["Arn"], - # ) - # envvars = generate_resource_envvars( - # bucket_name, - # xresources[bucket_name], - # S3_BUCKET_NAME, - # arn=found_bucket["Name"], - # ) - # apply_iam_based_resources( - # bucket, - # services_families, - # services_stack, - # res_root_stack, - # envvars, - # perms, - # ) diff --git a/ecs_composex/s3/s3_perms.json b/ecs_composex/s3/s3_perms.json index 5f37645e3..a6e18130f 100644 --- a/ecs_composex/s3/s3_perms.json +++ b/ecs_composex/s3/s3_perms.json @@ -1,27 +1,74 @@ { - "RWObjects": { - "Action": [ - "s3:GetObject*", - "s3:PutObject*" - ], - "Effect": "Allow" + "objects": { + "RW": { + "Action": [ + "s3:GetObject*", + "s3:PutObject*" + ], + "Effect": "Allow" + }, + "StrictRW": { + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow" + }, + "StrictRWDelete": { + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Effect": "Allow" + }, + "RWDelete": { + "Action": [ + "s3:GetObject*", + "s3:PutObject*", + "s3:DeleteObject*" + ], + "Effect": "Allow" + }, + "ReadOnly": { + "Action": [ + "s3:GetObject*" + ], + "Effect": "Allow" + }, + "StrictReadOnly": { + "Action": [ + "s3:GetObject" + ], + "Effect": "Allow" + }, + "WriteOnly": { + "Action": [ + "s3:PutObject*" + ], + "Effect": "Allow" + }, + "StrictWriteOnly": { + "Action": [ + "s3:PutObject" + ], + "Effect": "Allow" + } }, - "ReadOnlyObjects": { - "Action": [ - "s3:GetObject*" - ], - "Effect": "Allow" - }, - "WriteOnlyObjects": { - "Action": [ - "s3:PutObject*" - ], - "Effect": "Allow" - }, - "PowerUser": { - "NotAction": [ - "s3:CreateBucket", - "s3:DeleteBucket" - ] + "bucket": { + "ListOnly": { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ] + }, + "PowerUser": { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucket*", + "s3:SetBucket*" + ] + } } } diff --git a/ecs_composex/s3/s3_perms.py b/ecs_composex/s3/s3_perms.py index f188eac9d..276bcc260 100644 --- a/ecs_composex/s3/s3_perms.py +++ b/ecs_composex/s3/s3_perms.py @@ -36,6 +36,9 @@ def get_access_types(): return loads(perms_fd.read()) +ACCESS_TYPES = get_access_types() + + def generate_s3_bucket_resource_strings(res_name, attribute): """ Function to generate the SSM and CFN import/export strings diff --git a/use-cases/s3/lookup_use_create_buckets.yml b/use-cases/s3/lookup_use_create_buckets.yml index ee8a7cf3e..efaa731ed 100644 --- a/use-cases/s3/lookup_use_create_buckets.yml +++ b/use-cases/s3/lookup_use_create_buckets.yml @@ -7,7 +7,9 @@ x-s3: Settings: {} Services: - name: app03 - access: RWObjects + access: + bucket: ListOnly + objects: RW bucket-02: Lookup: @@ -16,4 +18,6 @@ x-s3: - aws:cloudformation:stack-name: pipeline-shared-buckets Services: - name: app03 - access: RWObjects + access: + bucket: PowerUser + objects: RW From c6b78d98501c2e52deeb720e091487d447935c68 Mon Sep 17 00:00:00 2001 From: John Preston Date: Sat, 17 Oct 2020 09:57:34 +0100 Subject: [PATCH 05/13] Working bucket and key finding. Todo: bucket created to services --- ecs_composex/resource_permissions.py | 2 +- ecs_composex/resource_settings.py | 2 +- ecs_composex/s3/s3_aws.py | 2 +- ecs_composex/s3/s3_ecs.py | 19 ++++---- ecs_composex/s3/s3_stack.py | 30 ++++++------ ecs_composex/s3/s3_template.py | 54 +++++++++------------- features/features/s3.feature | 18 ++++++++ use-cases/s3/lookup_use_create_buckets.yml | 11 +++++ 8 files changed, 76 insertions(+), 62 deletions(-) create mode 100644 features/features/s3.feature diff --git a/ecs_composex/resource_permissions.py b/ecs_composex/resource_permissions.py index 4684a481d..c95f1a915 100644 --- a/ecs_composex/resource_permissions.py +++ b/ecs_composex/resource_permissions.py @@ -19,7 +19,7 @@ Module to handle permissions from x-resource to ECS service """ -from ecs_composex.common import NONALPHANUM, LOG +from ecs_composex.common import NONALPHANUM, LOG, keyisset from ecs_composex.ecs.ecs_container_config import extend_container_envvars from ecs_composex.ecs.ecs_iam import define_service_containers from ecs_composex.ecs.ecs_params import TASK_ROLE_T diff --git a/ecs_composex/resource_settings.py b/ecs_composex/resource_settings.py index 58964a294..642f1b755 100644 --- a/ecs_composex/resource_settings.py +++ b/ecs_composex/resource_settings.py @@ -23,7 +23,7 @@ from troposphere import Sub, ImportValue from troposphere.iam import Policy as IamPolicy -from ecs_composex.common import LOG +from ecs_composex.common import LOG, keyisset from ecs_composex.common.cfn_params import ROOT_STACK_NAME_T from ecs_composex.common.ecs_composex import CFN_EXPORT_DELIMITER as DELIM diff --git a/ecs_composex/s3/s3_aws.py b/ecs_composex/s3/s3_aws.py index 1327699cc..1c66dcb02 100644 --- a/ecs_composex/s3/s3_aws.py +++ b/ecs_composex/s3/s3_aws.py @@ -58,7 +58,7 @@ def return_bucket_config(bucket_arn, session): == "ServerSideEncryptionConfigurationNotFoundError" ): raise - LOG.error(error.response["Error"]["Message"]) + LOG.warn(error.response["Error"]["Message"]) return bucket_config except client.exceptions.NoSuchBucket: return None diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py index 48783eca2..e9a54e08b 100644 --- a/ecs_composex/s3/s3_ecs.py +++ b/ecs_composex/s3/s3_ecs.py @@ -32,7 +32,6 @@ from ecs_composex.resource_settings import ( generate_resource_envvars, generate_resource_permissions, - validate_lookup_resource, ) from ecs_composex.ecs.ecs_template import get_service_family_name from ecs_composex.s3.s3_params import S3_BUCKET_NAME @@ -146,12 +145,13 @@ def define_bucket_access(bucket, access, service_template, service_family, famil :param bucket: :param access: + :param troposphere.Template service_template: + :param str service_family: + :param bool family_wide: :return: """ bucket_key = "bucket" objects_key = "objects" - bucket_perms = None - objects_perms = None if isinstance(access, str): LOG.warn( "For s3 buckets, you should define a dict for access, with bucket and/or object policies separate." @@ -238,9 +238,7 @@ def assign_lookup_buckets(bucket, mappings, service, services_stack, services_fa ) -def s3_to_ecs( - xresources, services_stack, services_families, res_root_stack, settings, **kwargs -): +def s3_to_ecs(xresources, services_stack, services_families, res_root_stack, settings): """ Function to handle permissions assignment to ECS services. @@ -249,7 +247,6 @@ def s3_to_ecs( :param services_families: services families :param ecs_composex.common.stack.ComposeXStack res_root_stack: s3 root stack :param ecs_composex.common.settings.ComposeXSettings settings: ComposeX Settings for execution - :param dict kwargs: :return: """ buckets_mappings = {} @@ -265,10 +262,10 @@ def s3_to_ecs( define_bucket_mappings(buckets_mappings, lookup_buckets, settings) LOG.debug(dumps(buckets_mappings, indent=4)) for res in new_buckets: - print(f"New resource to create {res.logical_name}") - handle_new_buckets( - xresources, services_families, services_stack, res_root_stack, l_buckets - ) + LOG.debug(f"Creating {res.name} as {res.logical_name}") + # handle_new_buckets( + # xresources, services_families, services_stack, res_root_stack, l_buckets + # ) for res in lookup_buckets: for service_def in res.services: assign_lookup_buckets( diff --git a/ecs_composex/s3/s3_stack.py b/ecs_composex/s3/s3_stack.py index be7886a7b..e33a7b129 100644 --- a/ecs_composex/s3/s3_stack.py +++ b/ecs_composex/s3/s3_stack.py @@ -26,7 +26,7 @@ from ecs_composex.common.stacks import ComposeXStack from ecs_composex.common.outputs import ComposeXOutput from ecs_composex.common.compose_resources import XResource, set_resources -from ecs_composex.s3.s3_params import RES_KEY, S3_BUCKET_NAME_T +from ecs_composex.s3.s3_params import RES_KEY, S3_BUCKET_NAME, S3_BUCKET_ARN from ecs_composex.s3.s3_template import generate_bucket @@ -43,28 +43,26 @@ def create_s3_template(settings): mono_template = False if not keyisset(RES_KEY, settings.compose_content): return None - buckets = settings.compose_content[RES_KEY] - if not [ - buckets[bucket_name] - for bucket_name in buckets - if buckets[bucket_name].properties - ]: + xresources = settings.compose_content[RES_KEY] + new_buckets = [ + xresources[bucket_name] + for bucket_name in xresources + if xresources[bucket_name].properties + ] + if not new_buckets: LOG.info("There are no buckets to create.") return None - if len(list(buckets.keys())) <= CFN_MAX_OUTPUTS: + if len(list(new_buckets)) <= CFN_MAX_OUTPUTS: mono_template = True template = build_template(f"S3 root by ECS ComposeX for {settings.name}") - for bucket_name in buckets: - bucket_res_name = NONALPHANUM.sub("", bucket_name) - bucket = generate_bucket( - bucket_name, bucket_res_name, buckets[bucket_name], settings - ) + for bucket in new_buckets: + bucket = generate_bucket(bucket, settings) if bucket: values = [ - (S3_BUCKET_NAME_T, "Arn", GetAtt(bucket, "Arn")), - (S3_BUCKET_NAME_T, "Name", Ref(bucket)), + (S3_BUCKET_ARN, S3_BUCKET_ARN.title, GetAtt(bucket, "Arn")), + (S3_BUCKET_NAME, S3_BUCKET_NAME.title, Ref(bucket)), ] outputs = ComposeXOutput(bucket, values, True) if mono_template: @@ -77,7 +75,7 @@ def create_s3_template(settings): bucket_template.add_resource(bucket) bucket_template.add_output(outputs.outputs) bucket_stack = ComposeXStack( - bucket_res_name, stack_template=bucket_template + bucket.logical_name, stack_template=bucket_template ) template.add_resource(bucket_stack) return template diff --git a/ecs_composex/s3/s3_template.py b/ecs_composex/s3/s3_template.py index 3dd3981be..a65bc5fcf 100644 --- a/ecs_composex/s3/s3_template.py +++ b/ecs_composex/s3/s3_template.py @@ -100,33 +100,34 @@ def define_accelerate_config(properties, settings): return config -def define_bucket(bucket_name, res_name, definition): +def define_bucket(bucket): """ Function to generate the S3 bucket object - :param bucket_name: - :param res_name: + :param ecs_composex.s3.s3_stack.Bucket bucket: :param definition: :return: """ - properties = definition["Properties"] if keyisset("Properties", definition) else {} - settings = definition["Settings"] if keyisset("Settings", definition) else {} props = { - "AccelerateConfiguration": define_accelerate_config(properties, settings), + "AccelerateConfiguration": define_accelerate_config( + bucket.properties, bucket.settings + ), "AccessControl": s3.BucketOwnerFullControl - if not keyisset("AccessControl", properties) - else properties["AccessControl"], - "BucketEncryption": handle_bucket_encryption(definition, settings), - "BucketName": definition["BucketName"] - if keyisset("BucketName", definition) + if not keyisset("AccessControl", bucket.properties) + else bucket.properties["AccessControl"], + "BucketEncryption": handle_bucket_encryption( + bucket.properties, bucket.settings + ), + "BucketName": bucket.properties["BucketName"] + if keyisset("BucketName", bucket.properties) else Ref(AWS_NO_VALUE), "ObjectLockEnabled": False - if not keyisset("ObjectLockEnabled", properties) - else properties["ObjectLockEnabled"], + if not keyisset("ObjectLockEnabled", bucket.properties) + else bucket.properties["ObjectLockEnabled"], "PublicAccessBlockConfiguration": s3.PublicAccessBlockConfiguration( - **properties["PublicAccessBlockConfiguration"] + **bucket.properties["PublicAccessBlockConfiguration"] ) - if keyisset("PublicAccessBlockConfiguration", properties) + if keyisset("PublicAccessBlockConfiguration", bucket.properties) else s3.PublicAccessBlockConfiguration( BlockPublicAcls=True, BlockPublicPolicy=True, @@ -134,34 +135,23 @@ def define_bucket(bucket_name, res_name, definition): RestrictPublicBuckets=True, ), "VersioningConfiguration": s3.VersioningConfiguration( - Status=properties["VersioningConfiguration"]["Status"] + Status=bucket.properties["VersioningConfiguration"]["Status"] ) - if keyisset("VersioningConfiguration", properties) + if keyisset("VersioningConfiguration", bucket.properties) else Ref(AWS_NO_VALUE), "Metadata": metadata, } - bucket = s3.Bucket(res_name, **props) + bucket = s3.Bucket(bucket.logical_name, **props) return bucket -def generate_bucket(bucket_name, bucket_res_name, bucket_definition, settings): +def generate_bucket(bucket, settings): """ Function to identify whether create new bucket or lookup for existing bucket - :param bucket_name: - :param bucket_res_name: - :param bucket_definition: + :param ecs_composex.s3.s3_stack.Bucket bucket: :param settings: :return: """ - if keyisset("Lookup", bucket_definition): - LOG.info("If bucket is found, its ARN will be added to the task") - return - elif keyisset("Use", bucket_definition): - LOG.info(f"Assuming bucket {bucket_name} exists to use.") - return - if not keyisset("Properties", bucket_definition): - LOG.warning(f"Properties for bucket {bucket_name} were not defined. Skipping") - return - bucket = define_bucket(bucket_name, bucket_res_name, bucket_definition) + bucket = define_bucket(bucket) return bucket diff --git a/features/features/s3.feature b/features/features/s3.feature new file mode 100644 index 000000000..1b0ff3f55 --- /dev/null +++ b/features/features/s3.feature @@ -0,0 +1,18 @@ +Feature: ecs_composex.s3 + + @s3 + Scenario Outline: New s3 buckets and services + Given I use as my docker-compose file and as override file + Then I render the docker-compose to composex to validate + + Examples: + | file_path | override_file | + | use-cases/blog.yml | use-cases/s3/simple_s3_bucket.yml | + + Scenario Outline: New and lookup s3 buckets and services + Given I use as my docker-compose file and as override file + Then I render the docker-compose to composex to validate + + Examples: + | file_path | override_file | + | use-cases/blog.yml | use-cases/s3/lookup_use_create_buckets.yml | diff --git a/use-cases/s3/lookup_use_create_buckets.yml b/use-cases/s3/lookup_use_create_buckets.yml index efaa731ed..f107ba5fd 100644 --- a/use-cases/s3/lookup_use_create_buckets.yml +++ b/use-cases/s3/lookup_use_create_buckets.yml @@ -21,3 +21,14 @@ x-s3: access: bucket: PowerUser objects: RW + + bucket-03: + Lookup: + Name: sacrificial-lamb + Tags: + - composex: "True" + Services: + - name: app03 + access: + bucket: PowerUser + objects: RW From 77fca74d7a377fdc3aeaf2409108a93e34187f54 Mon Sep 17 00:00:00 2001 From: John Preston Date: Sun, 18 Oct 2020 17:10:05 +0100 Subject: [PATCH 06/13] Working permissions assignment from bucket to ECS services --- ecs_composex/s3/s3_ecs.py | 135 +++++++++++++++++++++++++++--------- ecs_composex/s3/s3_perms.py | 75 +------------------- 2 files changed, 102 insertions(+), 108 deletions(-) diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py index e9a54e08b..b7df86ff3 100644 --- a/ecs_composex/s3/s3_ecs.py +++ b/ecs_composex/s3/s3_ecs.py @@ -32,57 +32,120 @@ from ecs_composex.resource_settings import ( generate_resource_envvars, generate_resource_permissions, + generate_export_strings, ) from ecs_composex.ecs.ecs_template import get_service_family_name -from ecs_composex.s3.s3_params import S3_BUCKET_NAME -from ecs_composex.s3.s3_perms import ACCESS_TYPES, generate_s3_permissions +from ecs_composex.s3.s3_params import S3_BUCKET_NAME, S3_BUCKET_ARN +from ecs_composex.s3.s3_perms import ACCESS_TYPES from ecs_composex.s3.s3_aws import lookup_bucket_config from ecs_composex.kms.kms_perms import ACCESS_TYPES as KMS_ACCESS_TYPES +def assign_service_permissions_to_bucket( + bucket, access, service_template, service_family, family_wide +): + bucket_key = "bucket" + objects_key = "objects" + bucket_arn_import = generate_export_strings(bucket.logical_name, S3_BUCKET_ARN) + if keyisset(bucket_key, access): + bucket_perms = generate_resource_permissions( + f"BucketAccess{bucket.logical_name}", + ACCESS_TYPES[bucket_key], + None, + arn=bucket_arn_import, + ) + add_iam_policy_to_service_task_role( + service_template, + bucket, + bucket_perms, + access[bucket_key], + service_family, + family_wide, + ) + if keyisset(objects_key, access): + objects_perms = generate_resource_permissions( + f"ObjectsAccess{bucket.logical_name}", + ACCESS_TYPES[objects_key], + None, + arn=Sub("${BucketArn}/*", BucketArn=bucket_arn_import), + ) + add_iam_policy_to_service_task_role( + service_template, + bucket, + objects_perms, + access[objects_key], + service_family, + family_wide, + ) + + +def assign_new_bucket_to_services( + bucket, services_stack, services_families, res_root_stack +): + """ + Function to assign the bucket services permissions to access the s3 bucket. + :param bucket: + :param services_stack: + :param services_families: + :param res_root_stack: + :return: + """ + bucket_key = "bucket" + objects_key = "objects" + access = {objects_key: "RW", bucket_key: "ListOnly"} + for service in bucket.services: + if not keyisset("access", service) or isinstance(service["access"], str): + LOG.warn( + f"No permissions associated for {service['name']}. Setting default." + ) + else: + access = service["access"] + service_family = get_service_family_name(services_families, service["name"]) + if service_family not in services_stack.stack_template.resources: + raise AttributeError( + f"No service {service_family} present in services stack" + ) + family_wide = True if service["name"] in services_families else False + service_stack = services_stack.stack_template.resources[service_family] + service_template = service_stack.stack_template + assign_service_permissions_to_bucket( + bucket, access, service_template, service_family, family_wide + ) + + def handle_new_buckets( - xresources, + resource, services_families, services_stack, res_root_stack, - l_buckets, nested=False, ): - buckets_r = [] + """ + + :param resource: The resource + :type resource: ecs_composex.s3.s3_stack.Bucket + :param services_families: + :param services_stack: + :param res_root_stack: + :param nested: + :return: + """ if res_root_stack.is_void: return s_resources = res_root_stack.stack_template.resources for resource_name in s_resources: - if isinstance(s_resources[resource_name], Bucket): - buckets_r.append(s_resources[resource_name].title) - elif issubclass(type(s_resources[resource_name]), ComposeXStack): + if issubclass(type(s_resources[resource_name]), ComposeXStack): handle_new_buckets( - xresources, + resource, services_families, services_stack, s_resources[resource_name], - l_buckets, nested=True, ) - - for bucket_name in xresources: - if bucket_name in buckets_r or NONALPHANUM.sub("", bucket_name) in buckets_r: - perms = generate_s3_permissions( - NONALPHANUM.sub("", bucket_name), S3_BUCKET_NAME - ) - envvars = generate_resource_envvars( - bucket_name, xresources[bucket_name], S3_BUCKET_NAME - ) - apply_iam_based_resources( - xresources[bucket_name], - services_families, - services_stack, - res_root_stack, - envvars, - perms, - nested, + else: + assign_new_bucket_to_services( + resource, services_stack, services_families, res_root_stack ) - del l_buckets[bucket_name] def get_bucket_kms_key_from_config(bucket_config): @@ -139,7 +202,9 @@ def define_bucket_mappings(buckets_mappings, buckets, settings): ) -def define_bucket_access(bucket, access, service_template, service_family, family_wide): +def define_lookup_buckets_access( + bucket, access, service_template, service_family, family_wide +): """ Function to create the IAM policy for the service access to bucket @@ -233,7 +298,7 @@ def assign_lookup_buckets(bucket, mappings, service, services_stack, services_fa if not keyisset("access", service): LOG.error(f"No access defined for s3 bucket {bucket.name}") return - define_bucket_access( + define_lookup_buckets_access( bucket, service["access"], service_template, service_family, family_wide ) @@ -250,7 +315,6 @@ def s3_to_ecs(xresources, services_stack, services_families, res_root_stack, set :return: """ buckets_mappings = {} - l_buckets = xresources.copy() new_buckets = [ xresources[name] for name in xresources @@ -263,9 +327,12 @@ def s3_to_ecs(xresources, services_stack, services_families, res_root_stack, set LOG.debug(dumps(buckets_mappings, indent=4)) for res in new_buckets: LOG.debug(f"Creating {res.name} as {res.logical_name}") - # handle_new_buckets( - # xresources, services_families, services_stack, res_root_stack, l_buckets - # ) + handle_new_buckets( + res, + services_families, + services_stack, + res_root_stack, + ) for res in lookup_buckets: for service_def in res.services: assign_lookup_buckets( diff --git a/ecs_composex/s3/s3_perms.py b/ecs_composex/s3/s3_perms.py index 276bcc260..e79e824f3 100644 --- a/ecs_composex/s3/s3_perms.py +++ b/ecs_composex/s3/s3_perms.py @@ -15,16 +15,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from os import path from json import loads - -from troposphere import Sub, ImportValue, Parameter -from troposphere.iam import Policy as IamPolicy - -from ecs_composex.common import LOG, NONALPHANUM -from ecs_composex.common.ecs_composex import CFN_EXPORT_DELIMITER as DELIM -from ecs_composex.common.cfn_params import ROOT_STACK_NAME -from ecs_composex.resource_settings import generate_export_strings +from os import path def get_access_types(): @@ -37,68 +29,3 @@ def get_access_types(): ACCESS_TYPES = get_access_types() - - -def generate_s3_bucket_resource_strings(res_name, attribute): - """ - Function to generate the SSM and CFN import/export strings - Returns the import in a tuple - - :param str res_name: name of the queue as defined in ComposeX File - :param str|Parameter attribute: The attribute to use in Import Name. - - :returns: ImportValue for CFN - :rtype: ImportValue - """ - if isinstance(attribute, str): - bucket_string = ( - f"${{{ROOT_STACK_NAME.title}}}{DELIM}{res_name}{DELIM}{attribute}" - ) - objects_string = ( - f"${{{ROOT_STACK_NAME.title}}}{DELIM}{res_name}{DELIM}{attribute}/*" - ) - elif isinstance(attribute, Parameter): - bucket_string = ( - f"${{{ROOT_STACK_NAME.title}}}{DELIM}{res_name}{DELIM}{attribute.title}" - ) - objects_string = ( - f"${{{ROOT_STACK_NAME.title}}}{DELIM}{res_name}{DELIM}{attribute.title}/*" - ) - else: - raise TypeError("Attribute can only be a string or Parameter") - - return [ImportValue(Sub(bucket_string)), ImportValue(Sub(objects_string))] - - -def generate_s3_permissions(resource_name, attribute, arn=None): - """ - Function to generate IAM permissions for a given x-resource. Returns the mapping of these for the given resource. - - :param str resource_name: The name of the resource - :param str attribute: the attribute of the resource we are using for Import - :param str arn: The ARN of the resource if already looked up. - :return: dict of the IAM policies associated with the resource. - :rtype dict: - """ - resource_policies = {} - policies = get_access_types() - for a_type in policies: - clean_policy = {"Version": "2012-10-17", "Statement": []} - LOG.debug(a_type) - policy_doc = policies[a_type].copy() - policy_doc["Sid"] = Sub(NONALPHANUM.sub("", f"{a_type}To{resource_name}")) - policy_doc["Resource"] = ( - generate_export_strings(resource_name, attribute) - if not arn - else [f"{arn}/*", arn] - ) - clean_policy["Statement"].append(policy_doc) - resource_policies[a_type] = IamPolicy( - PolicyName=Sub( - NONALPHANUM.sub( - "", f"{a_type}{resource_name}${{{ROOT_STACK_NAME.title}}}" - ) - ), - PolicyDocument=clean_policy, - ) - return resource_policies From dc3bbc88ab9684fc6a74f458298724d3caf8e088 Mon Sep 17 00:00:00 2001 From: John Preston Date: Sun, 18 Oct 2020 17:27:31 +0100 Subject: [PATCH 07/13] Fixing code smells and linting --- ecs_composex/ecs_composex.py | 53 +++++++++++++++++++--------- ecs_composex/resource_permissions.py | 2 +- ecs_composex/resource_settings.py | 3 +- ecs_composex/s3/s3_aws.py | 2 -- ecs_composex/s3/s3_ecs.py | 23 ++++++------ ecs_composex/s3/s3_stack.py | 10 +++--- ecs_composex/s3/s3_template.py | 7 ++-- 7 files changed, 58 insertions(+), 42 deletions(-) diff --git a/ecs_composex/ecs_composex.py b/ecs_composex/ecs_composex.py index 8b7813ff0..5e81c1558 100644 --- a/ecs_composex/ecs_composex.py +++ b/ecs_composex/ecs_composex.py @@ -242,13 +242,39 @@ def add_compute(root_template, settings, vpc_stack): return root_template.add_resource(compute_stack) +def handle_new_xstack( + key, + res_type, + settings, + services_families, + services_stack, + vpc_stack, + root_template, + xstack, +): + tcp_services = ["x-rds", "x-appmesh"] + if vpc_stack and key in tcp_services: + xstack.get_from_vpc_stack(vpc_stack) + elif not vpc_stack and key in tcp_services: + xstack.no_vpc_parameters() + LOG.debug(xstack, xstack.is_void) + if xstack.is_void: + invoke_x_to_ecs(res_type, settings, services_stack, services_families, xstack) + elif ( + hasattr(xstack, "title") + and hasattr(xstack, "stack_template") + and not xstack.is_void + ): + root_template.add_resource(xstack) + + def add_x_resources( root_template, settings, services_stack, services_families, vpc_stack=None ): """ Function to add each X resource from the compose file """ - tcp_services = ["x-rds", "x-appmesh"] + for key in settings.compose_content: if key.startswith(X_KEY) and key not in EXCLUDED_X_KEYS: res_type = RES_REGX.sub("", key) @@ -263,21 +289,16 @@ def add_x_resources( settings=settings, Parameters=parameters, ) - if vpc_stack and key in tcp_services: - xstack.get_from_vpc_stack(vpc_stack) - elif not vpc_stack and key in tcp_services: - xstack.no_vpc_parameters() - LOG.debug(xstack, xstack.is_void) - if xstack.is_void: - invoke_x_to_ecs( - res_type, settings, services_stack, services_families, xstack - ) - elif ( - hasattr(xstack, "title") - and hasattr(xstack, "stack_template") - and not xclass.is_void - ): - root_template.add_resource(xstack) + handle_new_xstack( + key, + res_type, + settings, + services_families, + services_stack, + vpc_stack, + root_template, + xstack, + ) def create_services(root_stack, settings, vpc_stack, dns_params, create_cluster): diff --git a/ecs_composex/resource_permissions.py b/ecs_composex/resource_permissions.py index c95f1a915..4684a481d 100644 --- a/ecs_composex/resource_permissions.py +++ b/ecs_composex/resource_permissions.py @@ -19,7 +19,7 @@ Module to handle permissions from x-resource to ECS service """ -from ecs_composex.common import NONALPHANUM, LOG, keyisset +from ecs_composex.common import NONALPHANUM, LOG from ecs_composex.ecs.ecs_container_config import extend_container_envvars from ecs_composex.ecs.ecs_iam import define_service_containers from ecs_composex.ecs.ecs_params import TASK_ROLE_T diff --git a/ecs_composex/resource_settings.py b/ecs_composex/resource_settings.py index 642f1b755..c98634008 100644 --- a/ecs_composex/resource_settings.py +++ b/ecs_composex/resource_settings.py @@ -22,6 +22,7 @@ from troposphere import Parameter from troposphere import Sub, ImportValue from troposphere.iam import Policy as IamPolicy +from troposphere.ecs import Environment from ecs_composex.common import LOG, keyisset from ecs_composex.common.cfn_params import ROOT_STACK_NAME_T @@ -137,7 +138,7 @@ def validate_lookup_resource(resource_name, resource_def, res_root_stack): f"{resource_name} is not created in ComposeX and does not have Lookup attribute" ) if ( - not resource_name in res_root_stack.stack_template.resources + resource_name not in res_root_stack.stack_template.resources and keyisset("Lookup", resource_def) and not keyisset("Tags", resource_def["Lookup"]) ): diff --git a/ecs_composex/s3/s3_aws.py b/ecs_composex/s3/s3_aws.py index 1c66dcb02..358d86de8 100644 --- a/ecs_composex/s3/s3_aws.py +++ b/ecs_composex/s3/s3_aws.py @@ -24,8 +24,6 @@ from botocore.exceptions import ClientError from ecs_composex.common import LOG, keyisset -from ecs_composex.common.aws import define_tagsgroups_filter_tags -from ecs_composex.s3.s3_params import S3_ARN_REGEX from ecs_composex.common.aws import find_aws_resource_arn_from_tags_api diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py index b7df86ff3..d67ea4cde 100644 --- a/ecs_composex/s3/s3_ecs.py +++ b/ecs_composex/s3/s3_ecs.py @@ -20,25 +20,23 @@ """ from json import dumps + from troposphere import FindInMap, Sub -from troposphere.s3 import Bucket -from ecs_composex.common import LOG, NONALPHANUM, keyisset +from ecs_composex.common import LOG, keyisset from ecs_composex.common.stacks import ComposeXStack +from ecs_composex.ecs.ecs_template import get_service_family_name +from ecs_composex.kms.kms_perms import ACCESS_TYPES as KMS_ACCESS_TYPES from ecs_composex.resource_permissions import ( - apply_iam_based_resources, add_iam_policy_to_service_task_role, ) from ecs_composex.resource_settings import ( - generate_resource_envvars, generate_resource_permissions, generate_export_strings, ) -from ecs_composex.ecs.ecs_template import get_service_family_name -from ecs_composex.s3.s3_params import S3_BUCKET_NAME, S3_BUCKET_ARN -from ecs_composex.s3.s3_perms import ACCESS_TYPES from ecs_composex.s3.s3_aws import lookup_bucket_config -from ecs_composex.kms.kms_perms import ACCESS_TYPES as KMS_ACCESS_TYPES +from ecs_composex.s3.s3_params import S3_BUCKET_ARN +from ecs_composex.s3.s3_perms import ACCESS_TYPES def assign_service_permissions_to_bucket( @@ -223,9 +221,12 @@ def define_lookup_buckets_access( " Using default RW Objects and ListBucket" ) access = {objects_key: "RW", bucket_key: "ListOnly"} - elif isinstance(access, dict): - if not keyisset(objects_key, access) or not keyisset(bucket_key, access): - raise KeyError("You must define at least bucket or object access") + elif ( + isinstance(access, dict) + and not keyisset(objects_key, access) + or not keyisset(bucket_key, access) + ): + raise KeyError("You must define at least bucket or object access") if keyisset(bucket_key, access): bucket_perms = generate_resource_permissions( f"BucketAccess{bucket.logical_name}", diff --git a/ecs_composex/s3/s3_stack.py b/ecs_composex/s3/s3_stack.py index e33a7b129..587517cbe 100644 --- a/ecs_composex/s3/s3_stack.py +++ b/ecs_composex/s3/s3_stack.py @@ -20,16 +20,14 @@ """ from troposphere import Ref, GetAtt -from troposphere.s3 import Bucket as S3Bucket -from ecs_composex.common import LOG, keyisset, build_template, NONALPHANUM -from ecs_composex.common.stacks import ComposeXStack -from ecs_composex.common.outputs import ComposeXOutput +from ecs_composex.common import LOG, keyisset, build_template from ecs_composex.common.compose_resources import XResource, set_resources +from ecs_composex.common.outputs import ComposeXOutput +from ecs_composex.common.stacks import ComposeXStack from ecs_composex.s3.s3_params import RES_KEY, S3_BUCKET_NAME, S3_BUCKET_ARN from ecs_composex.s3.s3_template import generate_bucket - CFN_MAX_OUTPUTS = 50 @@ -70,7 +68,7 @@ def create_s3_template(settings): template.add_output(outputs.outputs) elif not mono_template: bucket_template = build_template( - f"Template for DynamoDB bucket {bucket.title}" + f"Template for S3 Bucket {bucket.title}" ) bucket_template.add_resource(bucket) bucket_template.add_output(outputs.outputs) diff --git a/ecs_composex/s3/s3_template.py b/ecs_composex/s3/s3_template.py index a65bc5fcf..d17902d0f 100644 --- a/ecs_composex/s3/s3_template.py +++ b/ecs_composex/s3/s3_template.py @@ -15,13 +15,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from troposphere import s3 -from troposphere import Ref, GetAtt, Sub -from troposphere import AWS_NO_VALUE +from troposphere import Ref, s3, AWS_NO_VALUE -from ecs_composex.common import LOG, keyisset +from ecs_composex.common import keyisset from ecs_composex.s3 import metadata -from ecs_composex.s3.s3_params import S3_BUCKET_NAME def create_bucket_encryption_default(props=None): From c271679a0412286bef85fbe123d8685065751b21 Mon Sep 17 00:00:00 2001 From: John Preston Date: Mon, 19 Oct 2020 08:26:30 +0100 Subject: [PATCH 08/13] Restoring env vars and code linting --- ecs_composex/common/compose_resources.py | 6 +- ecs_composex/ecs_composex.py | 3 + ecs_composex/rds/rds_template.py | 2 +- ecs_composex/resource_settings.py | 70 +--------------------- ecs_composex/s3/s3_ecs.py | 9 ++- use-cases/s3/lookup_use_create_buckets.yml | 3 + use-cases/s3/simple_s3_bucket.yml | 4 ++ 7 files changed, 22 insertions(+), 75 deletions(-) diff --git a/ecs_composex/common/compose_resources.py b/ecs_composex/common/compose_resources.py index 87b65dc97..7bf215080 100644 --- a/ecs_composex/common/compose_resources.py +++ b/ecs_composex/common/compose_resources.py @@ -116,12 +116,14 @@ def __init__(self, name, definition): def __repr__(self): return self.logical_name - def generate_resource_envvars(self, attribute): + def generate_resource_envvars(self, attribute, arn=None): """ :return: environment key/pairs :rtype: list """ - export_string = generate_export_strings(self.logical_name, attribute) + export_string = ( + generate_export_strings(self.logical_name, attribute) if not arn else arn + ) if self.settings and keyisset("EnvNames", self.settings): for env_name in self.settings["EnvNames"]: self.env_vars.append( diff --git a/ecs_composex/ecs_composex.py b/ecs_composex/ecs_composex.py index 5e81c1558..3eb9259cd 100644 --- a/ecs_composex/ecs_composex.py +++ b/ecs_composex/ecs_composex.py @@ -186,6 +186,7 @@ def apply_x_configs_to_ecs( if ( issubclass(type(resource), ComposeXStack) and resource_name in SUPPORTED_X_MODULES + and not resource.is_void ): module = getattr(resource, "title") invoke_x_to_ecs( @@ -207,6 +208,7 @@ def apply_x_to_x_configs(root_template, settings): issubclass(type(resource), ComposeXStack) and resource_name in SUPPORTED_X_MODULES and hasattr(resource, "add_xdependencies") + and not resource.is_void ): resource.add_xdependencies(root_template, settings.compose_content) @@ -283,6 +285,7 @@ def add_x_resources( LOG.debug(xclass) if not xclass: LOG.info(f"Class for {res_type} not found") + xstack = None else: xstack = xclass( res_type.strip(), diff --git a/ecs_composex/rds/rds_template.py b/ecs_composex/rds/rds_template.py index 75eed1d4a..6d26da90e 100644 --- a/ecs_composex/rds/rds_template.py +++ b/ecs_composex/rds/rds_template.py @@ -21,7 +21,7 @@ from troposphere import Ref, Join -from ecs_composex.common import build_template, validate_kwargs, keyisset, LOG +from ecs_composex.common import build_template, validate_kwargs, LOG from ecs_composex.common.cfn_params import ROOT_STACK_NAME_T, ROOT_STACK_NAME from ecs_composex.common.stacks import ComposeXStack from ecs_composex.rds.rds_db_template import ( diff --git a/ecs_composex/resource_settings.py b/ecs_composex/resource_settings.py index c98634008..178aadd66 100644 --- a/ecs_composex/resource_settings.py +++ b/ecs_composex/resource_settings.py @@ -22,9 +22,8 @@ from troposphere import Parameter from troposphere import Sub, ImportValue from troposphere.iam import Policy as IamPolicy -from troposphere.ecs import Environment -from ecs_composex.common import LOG, keyisset +from ecs_composex.common import LOG from ecs_composex.common.cfn_params import ROOT_STACK_NAME_T from ecs_composex.common.ecs_composex import CFN_EXPORT_DELIMITER as DELIM @@ -78,70 +77,3 @@ def generate_resource_permissions(resource_name, policies, attribute, arn=None): PolicyDocument=clean_policy, ) return resource_policies - - -def generate_resource_envvars(resource_name, resource, attribute, arn=None): - """ - Function to generate environment variables that can be added to a container definition - shall the ecs_service need to know about the Queue - - :param str resource_name: The name of the resource - :param dict resource: The resource definition as defined in docker-compose file. - :param str attribute: the attribute of the resource we are using for Import - :param str arn: The ARN of the resource if already looked up. - - :return: environment key/pairs - :rtype: list - """ - env_names = [] - export_strings = ( - generate_export_strings(resource_name, attribute) if not arn else arn - ) - if keyisset("Settings", resource) and keyisset("EnvNames", resource["Settings"]): - for env_name in resource["Settings"]["EnvNames"]: - env_names.append( - Environment( - Name=env_name, - Value=export_strings, - ) - ) - if resource_name not in resource["Settings"]["EnvNames"]: - env_names.append( - Environment( - Name=resource_name, - Value=export_strings, - ) - ) - else: - env_names.append( - Environment( - Name=resource_name, - Value=export_strings, - ) - ) - return env_names - - -def validate_lookup_resource(resource_name, resource_def, res_root_stack): - """ - Function to validate a resource has attributes to lookup. - - :param str resource_name: - :param dict resource_def: - :param ecs_composex.common.stacks.ComposeXStack res_root_stack: - :return: - """ - if resource_name not in res_root_stack.stack_template.resources and not keyisset( - "Lookup", resource_def - ): - raise KeyError( - f"{resource_name} is not created in ComposeX and does not have Lookup attribute" - ) - if ( - resource_name not in res_root_stack.stack_template.resources - and keyisset("Lookup", resource_def) - and not keyisset("Tags", resource_def["Lookup"]) - ): - raise KeyError( - f"{resource_name} is defined for lookup but there are no tags indicated." - ) diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py index d67ea4cde..aab3aae28 100644 --- a/ecs_composex/s3/s3_ecs.py +++ b/ecs_composex/s3/s3_ecs.py @@ -35,7 +35,7 @@ generate_export_strings, ) from ecs_composex.s3.s3_aws import lookup_bucket_config -from ecs_composex.s3.s3_params import S3_BUCKET_ARN +from ecs_composex.s3.s3_params import S3_BUCKET_ARN, S3_BUCKET_NAME from ecs_composex.s3.s3_perms import ACCESS_TYPES @@ -45,6 +45,8 @@ def assign_service_permissions_to_bucket( bucket_key = "bucket" objects_key = "objects" bucket_arn_import = generate_export_strings(bucket.logical_name, S3_BUCKET_ARN) + bucket_name_import = generate_export_strings(bucket.logical_name, S3_BUCKET_NAME) + bucket.generate_resource_envvars(None, bucket_name_import) if keyisset(bucket_key, access): bucket_perms = generate_resource_permissions( f"BucketAccess{bucket.logical_name}", @@ -128,8 +130,6 @@ def handle_new_buckets( :param nested: :return: """ - if res_root_stack.is_void: - return s_resources = res_root_stack.stack_template.resources for resource_name in s_resources: if issubclass(type(s_resources[resource_name]), ComposeXStack): @@ -227,6 +227,9 @@ def define_lookup_buckets_access( or not keyisset(bucket_key, access) ): raise KeyError("You must define at least bucket or object access") + bucket.generate_resource_envvars( + None, arn=FindInMap("s3", bucket.logical_name, "Name") + ) if keyisset(bucket_key, access): bucket_perms = generate_resource_permissions( f"BucketAccess{bucket.logical_name}", diff --git a/use-cases/s3/lookup_use_create_buckets.yml b/use-cases/s3/lookup_use_create_buckets.yml index f107ba5fd..d285a3bb8 100644 --- a/use-cases/s3/lookup_use_create_buckets.yml +++ b/use-cases/s3/lookup_use_create_buckets.yml @@ -23,6 +23,9 @@ x-s3: objects: RW bucket-03: + Settings: + EnvNames: + - BUCKET03 Lookup: Name: sacrificial-lamb Tags: diff --git a/use-cases/s3/simple_s3_bucket.yml b/use-cases/s3/simple_s3_bucket.yml index 7d2f91a79..9e9e14c83 100644 --- a/use-cases/s3/simple_s3_bucket.yml +++ b/use-cases/s3/simple_s3_bucket.yml @@ -4,6 +4,10 @@ x-s3: bucket-01: Properties: BucketName: bucket-01 + Settings: + EnvNames: + - bucket01 + - BUCKET_ABCD-01 Services: - name: app03 access: RWObjects From dc92e9ce0b4b120c71ac408103fd66188787ec41 Mon Sep 17 00:00:00 2001 From: John Preston Date: Mon, 19 Oct 2020 10:00:58 +0100 Subject: [PATCH 09/13] Adding more configuration and testing --- ecs_composex/s3/s3_template.py | 160 ++++++++++++++++++++++-------- features/features/s3.feature | 8 ++ use-cases/s3/simple_s3_bucket.yml | 57 +++++++++++ 3 files changed, 183 insertions(+), 42 deletions(-) diff --git a/ecs_composex/s3/s3_template.py b/ecs_composex/s3/s3_template.py index d17902d0f..8c23148ee 100644 --- a/ecs_composex/s3/s3_template.py +++ b/ecs_composex/s3/s3_template.py @@ -17,7 +17,7 @@ from troposphere import Ref, s3, AWS_NO_VALUE -from ecs_composex.common import keyisset +from ecs_composex.common import keyisset, keypresent from ecs_composex.s3 import metadata @@ -34,6 +34,49 @@ def create_bucket_encryption_default(props=None): ) +def handle_encryption_settings(setting): + """ + Function to parse the macro-like settings for bucket creation + + :param bool,str setting: + :return: + """ + if ( + isinstance(setting, bool) + and setting is True + or isinstance(setting, str) + and setting == "AES256" + ): + return create_bucket_encryption_default() + + +def handle_encryption_rule(encryption_rules): + """ + Function to handle the Encryption rule for BucketEncryption + + :param list encryption_rules: + :return: + """ + if len(encryption_rules) != 1: + raise ValueError("There can only be one encryption rule") + prop_key = "ServerSideEncryptionByDefault" + rule = encryption_rules[0] + if keyisset("ServerSideEncryptionByDefault", rule): + if ( + keyisset("SSEAlgorithm", rule[prop_key]) + and rule[prop_key]["SSEAlgorithm"] == "AES256" + ): + return create_bucket_encryption_default() + elif ( + keyisset("SSEAlgorithm", rule[prop_key]) + and rule[prop_key]["SSEAlgorithm"] == "aws:kms" + ): + if not keyisset("KMSMasterKeyID", rule[prop_key]): + raise KeyError("Missing attribute KMSMasterKeyID for KMS Encryption") + else: + return create_bucket_encryption_default(rule[prop_key]) + + def handle_bucket_encryption(properties, settings): """ Function to handle the S3 bucket encryption. @@ -42,25 +85,25 @@ def handle_bucket_encryption(properties, settings): :param dict settings: :return: """ + settings_key = "EnableEncryption" default = create_bucket_encryption_default() if not keyisset("BucketEncryption", properties) and not keyisset( - "BucketEncryption", settings + settings_key, settings ): return default - elif keyisset("BucketEncryption", settings): - if ( - keyisset("SSEAlgorithm", settings["BucketEncryption"]) - and settings["BucketEncryption"]["SSEAlgorithm"] == "AES256" - ): - return default - elif ( - keyisset("SSEAlgorithm", settings["BucketEncryption"]) - and settings["BucketEncryption"]["SSEAlgorithm"] == "aws:kms" - ): - if not keyisset("KMSMasterKeyID", settings["BucketEncryption"]): - raise KeyError("Missing attribute KMSMasterKeyID for KMS Encryption") - else: - return create_bucket_encryption_default(settings["BucketEncryption"]) + elif keyisset(settings_key, settings): + return handle_encryption_settings(settings[settings_key]) + elif ( + keyisset("BucketEncryption", properties) + and keyisset( + "ServerSideEncryptionConfiguration", properties["BucketEncryption"] + ) + and properties["BucketEncryption"]["ServerSideEncryptionConfiguration"] + ): + handle_encryption_rule( + properties["BucketEncryption"]["ServerSideEncryptionConfiguration"] + ) + return create_bucket_encryption_default() def define_public_block_access(properties): @@ -69,6 +112,54 @@ def define_public_block_access(properties): :param properties: :return: """ + if keyisset("PublicAccessBlockConfiguration", properties): + return s3.PublicAccessBlockConfiguration( + **properties["PublicAccessBlockConfiguration"] + ) + + else: + return s3.PublicAccessBlockConfiguration( + BlockPublicAcls=True, + BlockPublicPolicy=True, + IgnorePublicAcls=True, + RestrictPublicBuckets=True, + ) + + +def define_objects_locking(properties): + """ + Function to define bucket objects lock + + :param dict properties: + :return: + """ + if not keypresent("ObjectLockEnabled", properties): + return False + else: + return properties["ObjectLockEnabled"] + + +def define_bucket_versioning(properties): + """ + Function to define bucket versioning + + :param dict properties: + :return: + """ + if keyisset("VersioningConfiguration", properties): + return s3.VersioningConfiguration( + Status=properties["VersioningConfiguration"]["Status"] + ) + + else: + return Ref(AWS_NO_VALUE) + + +def define_access_control(properties): + if not keyisset("AccessControl", properties): + return s3.BucketOwnerFullControl + else: + return properties["AccessControl"] def define_accelerate_config(properties, settings): @@ -90,10 +181,10 @@ def define_accelerate_config(properties, settings): properties["AccelerateConfiguration"]["AccelerationStatus"] ) ) - elif keyisset("AccelerationStatus", settings): - config = s3.AccelerateConfiguration( - AccelerationStatus=settings["AccelerationStatus"] - ) + elif keyisset("EnableAcceleration", settings) and isinstance( + settings["EnableAcceleration"], bool + ): + config = s3.AccelerateConfiguration(AccelerationStatus="Enabled") return config @@ -102,41 +193,26 @@ def define_bucket(bucket): Function to generate the S3 bucket object :param ecs_composex.s3.s3_stack.Bucket bucket: - :param definition: :return: """ props = { "AccelerateConfiguration": define_accelerate_config( bucket.properties, bucket.settings ), - "AccessControl": s3.BucketOwnerFullControl - if not keyisset("AccessControl", bucket.properties) - else bucket.properties["AccessControl"], "BucketEncryption": handle_bucket_encryption( bucket.properties, bucket.settings ), + "AccessControl": define_access_control(bucket.properties), "BucketName": bucket.properties["BucketName"] if keyisset("BucketName", bucket.properties) else Ref(AWS_NO_VALUE), - "ObjectLockEnabled": False - if not keyisset("ObjectLockEnabled", bucket.properties) - else bucket.properties["ObjectLockEnabled"], - "PublicAccessBlockConfiguration": s3.PublicAccessBlockConfiguration( - **bucket.properties["PublicAccessBlockConfiguration"] - ) - if keyisset("PublicAccessBlockConfiguration", bucket.properties) - else s3.PublicAccessBlockConfiguration( - BlockPublicAcls=True, - BlockPublicPolicy=True, - IgnorePublicAcls=True, - RestrictPublicBuckets=True, - ), - "VersioningConfiguration": s3.VersioningConfiguration( - Status=bucket.properties["VersioningConfiguration"]["Status"] - ) - if keyisset("VersioningConfiguration", bucket.properties) - else Ref(AWS_NO_VALUE), + "ObjectLockEnabled": define_objects_locking(bucket.properties), + "PublicAccessBlockConfiguration": define_public_block_access(bucket.properties), + "VersioningConfiguration": define_bucket_versioning(bucket.properties), "Metadata": metadata, + "DeletionPolicy": "Retain" + if not keyisset("DeletionPolicy", bucket.settings) + else bucket.settings["DeletionPolicy"], } bucket = s3.Bucket(bucket.logical_name, **props) return bucket diff --git a/features/features/s3.feature b/features/features/s3.feature index 1b0ff3f55..320ed0bb2 100644 --- a/features/features/s3.feature +++ b/features/features/s3.feature @@ -16,3 +16,11 @@ Feature: ecs_composex.s3 Examples: | file_path | override_file | | use-cases/blog.yml | use-cases/s3/lookup_use_create_buckets.yml | + + Scenario Outline: NLookup s3 buckets only + Given I use as my docker-compose file and as override file + Then I render the docker-compose to composex to validate + + Examples: + | file_path | override_file | + | use-cases/blog.yml | use-cases/s3/lookup_only.yml | diff --git a/use-cases/s3/simple_s3_bucket.yml b/use-cases/s3/simple_s3_bucket.yml index 9e9e14c83..a8be3e349 100644 --- a/use-cases/s3/simple_s3_bucket.yml +++ b/use-cases/s3/simple_s3_bucket.yml @@ -4,6 +4,23 @@ x-s3: bucket-01: Properties: BucketName: bucket-01 + AccessControl: BucketOwnerFullControl + ObjectLockEnabled: True + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: False + AccelerateConfiguration: + AccelerateStatus: Suspended + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: "aws:kms" + KMSMasterKeyID: "aws/s3" + VersioningConfiguration: + Status: "Enabled" + Settings: EnvNames: - bucket01 @@ -11,3 +28,43 @@ x-s3: Services: - name: app03 access: RWObjects + bucket-03: + Properties: + BucketName: bucket-01 + AccessControl: BucketOwnerFullControl + ObjectLockEnabled: True + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: False + AccelerateConfiguration: + AccelerateStatus: Suspended + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + VersioningConfiguration: + Status: "Enabled" + + Settings: + EnvNames: + - bucket01 + - BUCKET_ABCD-01 + Services: + - name: app03 + access: RWObjects + bucket-02: + Properties: + BucketName: bucket-01 + Settings: + EnableEncryption: AES256 + EnableAcceleration: True + EnvNames: + - bucket01 + - BUCKET_ABCD-01 + Services: + - name: app03 + access: + bucket: ListOnly + objects: RW From a5f4ad7b79b591b7f3e614a7dcf41f113c91eeb1 Mon Sep 17 00:00:00 2001 From: John Preston Date: Mon, 19 Oct 2020 10:27:24 +0100 Subject: [PATCH 10/13] Update changes to adapt to empty properties --- ecs_composex/common/compose_resources.py | 15 +++++++----- ecs_composex/s3/s3_ecs.py | 4 +--- ecs_composex/s3/s3_stack.py | 2 +- use-cases/s3/lookup_only.yml | 27 ++++++++++++++++++++++ use-cases/s3/lookup_use_create_buckets.yml | 3 +-- 5 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 use-cases/s3/lookup_only.yml diff --git a/ecs_composex/common/compose_resources.py b/ecs_composex/common/compose_resources.py index 7bf215080..9aa7dc5e8 100644 --- a/ecs_composex/common/compose_resources.py +++ b/ecs_composex/common/compose_resources.py @@ -22,7 +22,7 @@ from troposphere import Sub from troposphere.ecs import Environment -from ecs_composex.common import LOG, NONALPHANUM, keyisset +from ecs_composex.common import LOG, NONALPHANUM, keyisset, keypresent from ecs_composex.resource_settings import generate_export_strings from ecs_composex.common.cfn_params import ROOT_STACK_NAME @@ -93,11 +93,14 @@ def __init__(self, name, definition): if not keyisset("Settings", self.definition) else self.definition["Settings"] ) - self.properties = ( - None - if not keyisset("Properties", self.definition) - else self.definition["Properties"] - ) + if keyisset("Properties", self.definition): + self.properties = self.definition["Properties"] + elif not keyisset("Properties", self.definition) and keypresent( + "Properties", self.definition + ): + self.properties = {} + else: + self.properties = None self.services = ( [] if not keyisset("Services", self.definition) diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py index aab3aae28..f6a7d13e4 100644 --- a/ecs_composex/s3/s3_ecs.py +++ b/ecs_composex/s3/s3_ecs.py @@ -320,9 +320,7 @@ def s3_to_ecs(xresources, services_stack, services_families, res_root_stack, set """ buckets_mappings = {} new_buckets = [ - xresources[name] - for name in xresources - if xresources[name].properties and not xresources[name].lookup + xresources[name] for name in xresources if not xresources[name].lookup ] lookup_buckets = [ xresources[name] for name in xresources if xresources[name].lookup diff --git a/ecs_composex/s3/s3_stack.py b/ecs_composex/s3/s3_stack.py index 587517cbe..9e8c7c68d 100644 --- a/ecs_composex/s3/s3_stack.py +++ b/ecs_composex/s3/s3_stack.py @@ -45,7 +45,7 @@ def create_s3_template(settings): new_buckets = [ xresources[bucket_name] for bucket_name in xresources - if xresources[bucket_name].properties + if not xresources[bucket_name].lookup ] if not new_buckets: LOG.info("There are no buckets to create.") diff --git a/use-cases/s3/lookup_only.yml b/use-cases/s3/lookup_only.yml new file mode 100644 index 000000000..e9228694c --- /dev/null +++ b/use-cases/s3/lookup_only.yml @@ -0,0 +1,27 @@ +version: "3.8" + +x-s3: + bucket-02: + Lookup: + Tags: + - aws:cloudformation:logical-id: ArtifactsBucket + - aws:cloudformation:stack-name: pipeline-shared-buckets + Services: + - name: app03 + access: + bucket: PowerUser + objects: RW + + bucket-03: + Settings: + EnvNames: + - BUCKET03 + Lookup: + Name: sacrificial-lamb + Tags: + - composex: "True" + Services: + - name: app03 + access: + bucket: PowerUser + objects: RW diff --git a/use-cases/s3/lookup_use_create_buckets.yml b/use-cases/s3/lookup_use_create_buckets.yml index d285a3bb8..66f39744e 100644 --- a/use-cases/s3/lookup_use_create_buckets.yml +++ b/use-cases/s3/lookup_use_create_buckets.yml @@ -2,8 +2,7 @@ x-s3: bucket-01: - Properties: - BucketName: bucket-01 + Properties: {} Settings: {} Services: - name: app03 From 9577005bdd951cf0bd97183aef46ed15a733ab55 Mon Sep 17 00:00:00 2001 From: John Preston Date: Mon, 19 Oct 2020 11:16:57 +0100 Subject: [PATCH 11/13] Adding settings for bucket name expansion --- ecs_composex/s3/s3_stack.py | 2 +- ecs_composex/s3/s3_template.py | 45 ++++++++++++++++++++++++++----- use-cases/s3/simple_s3_bucket.yml | 26 ++++++++++++++++-- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/ecs_composex/s3/s3_stack.py b/ecs_composex/s3/s3_stack.py index 9e8c7c68d..3e5e43076 100644 --- a/ecs_composex/s3/s3_stack.py +++ b/ecs_composex/s3/s3_stack.py @@ -56,7 +56,7 @@ def create_s3_template(settings): template = build_template(f"S3 root by ECS ComposeX for {settings.name}") for bucket in new_buckets: - bucket = generate_bucket(bucket, settings) + bucket = generate_bucket(bucket) if bucket: values = [ (S3_BUCKET_ARN, S3_BUCKET_ARN.title, GetAtt(bucket, "Arn")), diff --git a/ecs_composex/s3/s3_template.py b/ecs_composex/s3/s3_template.py index 8c23148ee..4b9843f32 100644 --- a/ecs_composex/s3/s3_template.py +++ b/ecs_composex/s3/s3_template.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from troposphere import Ref, s3, AWS_NO_VALUE +from troposphere import Ref, Sub, s3, AWS_NO_VALUE -from ecs_composex.common import keyisset, keypresent +from ecs_composex.common import keyisset, keypresent, LOG from ecs_composex.s3 import metadata @@ -188,6 +188,41 @@ def define_accelerate_config(properties, settings): return config +def define_bucket_name(properties, settings): + expand_region_key = "ExpandRegionToBucket" + expand_account_id = "ExpandAccountIdToBucket" + base_name = ( + None if not keyisset("BucketName", properties) else properties["BucketName"] + ) + print( + base_name, + keyisset(expand_region_key, settings), + keyisset(expand_account_id, settings), + ) + if base_name: + if keyisset(expand_region_key, settings) and keyisset( + expand_account_id, settings + ): + return Sub(f"{base_name}.${{AWS::AccountId}}.${{AWS::Region}}") + elif keyisset(expand_region_key, settings) and not keyisset( + expand_account_id, settings + ): + return Sub(f"{base_name}.${{AWS::Region}}") + elif not keyisset(expand_region_key, settings) and keyisset( + expand_account_id, settings + ): + return Sub(f"{base_name}.${{AWS::AccountId}}") + elif not keyisset(expand_account_id, settings) and not keyisset( + expand_region_key, settings + ): + LOG.warn( + f"{base_name} - You defined the bucket without any extension. " + "Bucket names must be unique. Make sure it is not already in-use" + ) + return base_name + return Ref(AWS_NO_VALUE) + + def define_bucket(bucket): """ Function to generate the S3 bucket object @@ -203,9 +238,7 @@ def define_bucket(bucket): bucket.properties, bucket.settings ), "AccessControl": define_access_control(bucket.properties), - "BucketName": bucket.properties["BucketName"] - if keyisset("BucketName", bucket.properties) - else Ref(AWS_NO_VALUE), + "BucketName": define_bucket_name(bucket.properties, bucket.settings), "ObjectLockEnabled": define_objects_locking(bucket.properties), "PublicAccessBlockConfiguration": define_public_block_access(bucket.properties), "VersioningConfiguration": define_bucket_versioning(bucket.properties), @@ -218,7 +251,7 @@ def define_bucket(bucket): return bucket -def generate_bucket(bucket, settings): +def generate_bucket(bucket): """ Function to identify whether create new bucket or lookup for existing bucket diff --git a/use-cases/s3/simple_s3_bucket.yml b/use-cases/s3/simple_s3_bucket.yml index a8be3e349..96c933e63 100644 --- a/use-cases/s3/simple_s3_bucket.yml +++ b/use-cases/s3/simple_s3_bucket.yml @@ -22,6 +22,8 @@ x-s3: Status: "Enabled" Settings: + ExpandRegionToBucket: True + ExpandAccountIdToBucket: True EnvNames: - bucket01 - BUCKET_ABCD-01 @@ -30,7 +32,7 @@ x-s3: access: RWObjects bucket-03: Properties: - BucketName: bucket-01 + BucketName: bucket-03 AccessControl: BucketOwnerFullControl ObjectLockEnabled: True PublicAccessBlockConfiguration: @@ -48,6 +50,8 @@ x-s3: Status: "Enabled" Settings: + ExpandRegionToBucket: True + ExpandAccountIdToBucket: False EnvNames: - bucket01 - BUCKET_ABCD-01 @@ -55,9 +59,27 @@ x-s3: - name: app03 access: RWObjects bucket-02: + Properties: {} + Settings: + ExpandRegionToBucket: False + ExpandAccountIdToBucket: False + EnableEncryption: AES256 + EnableAcceleration: True + EnvNames: + - bucket01 + - BUCKET_ABCD-01 + Services: + - name: app03 + access: + bucket: ListOnly + objects: RW + + bucket-04: Properties: - BucketName: bucket-01 + BucketName: bucket-04 Settings: + ExpandRegionToBucket: False + ExpandAccountIdToBucket: False EnableEncryption: AES256 EnableAcceleration: True EnvNames: From 4fdca8e1044e71f5d4d84ff034202cab23649d2f Mon Sep 17 00:00:00 2001 From: John Preston Date: Mon, 19 Oct 2020 13:31:19 +0100 Subject: [PATCH 12/13] Adding docs --- docs/features.rst | 2 + docs/modules_syntax.rst | 2 + ecs_composex/s3/README.rst | 55 ++++++++++++++++++++ ecs_composex/s3/SYNTAX.rst | 102 +++++++++++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+) create mode 100644 ecs_composex/s3/README.rst create mode 100644 ecs_composex/s3/SYNTAX.rst diff --git a/docs/features.rst b/docs/features.rst index 31614b3c8..a706e3796 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -11,6 +11,8 @@ .. include:: ../ecs_composex/dynamodb/README.rst +.. include:: ../ecs_composex/s3/README.rst + .. include:: ../ecs_composex/vpc/README.rst .. include:: ../ecs_composex/kms/README.rst diff --git a/docs/modules_syntax.rst b/docs/modules_syntax.rst index c1c351c85..d37068d47 100644 --- a/docs/modules_syntax.rst +++ b/docs/modules_syntax.rst @@ -15,6 +15,8 @@ .. include:: ../ecs_composex/dynamodb/SYNTAX.rst +.. include:: ../ecs_composex/s3/SYNTAX.rst + .. include:: ../ecs_composex/secrets/SYNTAX.rst .. include:: ../ecs_composex/sns/SYNTAX.rst diff --git a/ecs_composex/s3/README.rst b/ecs_composex/s3/README.rst new file mode 100644 index 000000000..539ac5c7b --- /dev/null +++ b/ecs_composex/s3/README.rst @@ -0,0 +1,55 @@ +AWS S3 +======= + +This package is here to integrate AWS S3 buckets creation and association to ECS Services. + +Constraints +----------- + +S3 buckets are a delicate resource, mostly due to + +* Bucket names are within a global domain space, meaning, their can only be one bucket if a given name across all of AWS +* IAM permissions for buckets require to differentiate permissions to the bucket and to the objects +* Buckets also have policies, but we can't add a statement to the policy, one need to update the whole policy with the new statement + + +Settings +-------- + +AWS S3 bucket properties can be long and tedious to set correctly. To help with making your life easy, additional settings +have been added to shorten the bucket definition. + +* ExpandRegionToBucket +* ExpandAccountIdToBucket +* EnableEncryption + +Services +-------- + +The services work as usual, with a change syntax for the access, expecting a dictionary to distinguish Bucket and Objects +access, in order to provide the most tuned access to your services. + + +Features +-------- + +By default, if not specified, we have decided to encrypt files at rest with **AES256** SSEAlgorithm. The reason for that +choice is that, files are encrypted, for compliance, but without the complexity that KMS can bring and developers can +easily forget about. + +Also, objects are not locked, but, all public access is denied by default. You can obviously override these properties. + +Lookup +------ + +As for most resources now, you can lookup and find existing S3 buckets that you wish to manage outside of ComposeX. +So you can set tags to find your bucket (and its name, in which case it will cross-validate it). + +.. hint:: + + If your bucket is encrypted with a KMS key, the IAM task role for your service is also granted access to that Key + to manipulate the data in the bucket. + +.. tip:: + + For more details on the Settings and properties to set in your YAML Compose file, go to :ref:`s3_syntax_reference` diff --git a/ecs_composex/s3/SYNTAX.rst b/ecs_composex/s3/SYNTAX.rst new file mode 100644 index 000000000..2e0361362 --- /dev/null +++ b/ecs_composex/s3/SYNTAX.rst @@ -0,0 +1,102 @@ +.. _s3_syntax_reference: + +x-s3 +===== + +For the properties, go to to `AWS CFN S3 Definition`_ + +Services +-------- + +As for all other resource types, you can define the type of access you want based to the S3 buckets. +However, for buckets, this means distinguish the bucket and the objects resource. + +Access types +^^^^^^^^^^^^ + +.. literalinclude:: ../ecs_composex/s3/s3_perms.json + :language: json + +Settings +-------- + +Some use-cases require special adjustments. This is what this section is for. + +* `ExpandRegionToBucket`_ +* `ExpandAccountIdToBucket`_ +* `EnableEncryption`_ + +ExpandRegionToBucket +^^^^^^^^^^^^^^^^^^^^ + +When definining the `BucketName` in properties, if wanted to, for uniqueness or readability, you can append to that string +the region id (which is DNS compliant) to the bucket name. + +.. code-block:: yaml + + Properties: + BucketName: abcd-01 + Settings: + ExpandRegionToBucket: True + +Results into + +.. code-block:: yaml + + !Sub abcd-01.${AWS::Region} + +ExpandAccountIdToBucket +^^^^^^^^^^^^^^^^^^^^^^^ + +Similar to ExpandRegionToBucket, it will append the account ID (additional or instead of). + +.. code-block:: yaml + + Properties: + BucketName: abcd-01 + Settings: + ExpandRegionToBucket: True + +Results into + +.. code-block:: yaml + + !Sub 'abcd-01.${AWS::AccountId}' + +.. hint:: + + If you set both ExpandAccountIdToBucket and ExpandRegionToBucket, you end up with + + .. code-block:: yaml + + !Sub 'abcd-01.${AWS::Region}.${AWS::AccountId}' + +EnableEncryption +^^^^^^^^^^^^^^^^ + +If set to True (default) it will automatically define bucket encryption using AES256. + +.. hint:: + + Soon will link x-kms keys definition to that to allow you to re-use existing keys. + + +Lookup +------ + +The lookup allows you to find your cluster or db instance and also the Secret associated with them to allow ECS Services +to get access to these. + +It will also find the DB security group and add an ingress rule. + +.. code-block:: yaml + + x-s3: + bucket-01: + Lookup: + Name: my-long-complicated-bucket-name + Tags: + - sometag: value + + +.. _AWS CFN S3 Definition: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket.html From 03dbdca34264fe0c0923c278b45c1b8f616d53a8 Mon Sep 17 00:00:00 2001 From: John Preston Date: Mon, 19 Oct 2020 14:43:05 +0100 Subject: [PATCH 13/13] Updated doc, fixed a few things and tested deployment end-to-end --- ecs_composex/s3/SYNTAX.rst | 21 ++++++++++-- ecs_composex/s3/s3_ecs.py | 2 ++ ecs_composex/s3/s3_template.py | 53 +++++++++++++++++++++++++------ use-cases/s3/simple_s3_bucket.yml | 1 + 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/ecs_composex/s3/SYNTAX.rst b/ecs_composex/s3/SYNTAX.rst index 2e0361362..06f0fdc36 100644 --- a/ecs_composex/s3/SYNTAX.rst +++ b/ecs_composex/s3/SYNTAX.rst @@ -43,7 +43,7 @@ Results into .. code-block:: yaml - !Sub abcd-01.${AWS::Region} + !Sub abcd-01-${AWS::Region} ExpandAccountIdToBucket ^^^^^^^^^^^^^^^^^^^^^^^ @@ -61,7 +61,7 @@ Results into .. code-block:: yaml - !Sub 'abcd-01.${AWS::AccountId}' + !Sub 'abcd-01-${AWS::AccountId}' .. hint:: @@ -69,7 +69,22 @@ Results into .. code-block:: yaml - !Sub 'abcd-01.${AWS::Region}.${AWS::AccountId}' + !Sub 'abcd-01-${AWS::Region}-${AWS::AccountId}' + +NameSeparator +^^^^^^^^^^^^^ + +As shown above, the separator between the bucket name and AWS::AccountId or AWS::Region is **-**. This parameter allows +you to define something else. + +.. note:: + + I would recommend not more than 2 characters separator. + +.. warning:: + + The separator must allow for DNS compliance **[a-z0-9.-]** + EnableEncryption ^^^^^^^^^^^^^^^^ diff --git a/ecs_composex/s3/s3_ecs.py b/ecs_composex/s3/s3_ecs.py index f6a7d13e4..f7008c665 100644 --- a/ecs_composex/s3/s3_ecs.py +++ b/ecs_composex/s3/s3_ecs.py @@ -111,6 +111,8 @@ def assign_new_bucket_to_services( assign_service_permissions_to_bucket( bucket, access, service_template, service_family, family_wide ) + if res_root_stack.title not in services_stack.DependsOn: + services_stack.DependsOn.append(res_root_stack.title) def handle_new_buckets( diff --git a/ecs_composex/s3/s3_template.py b/ecs_composex/s3/s3_template.py index 4b9843f32..5ddc14396 100644 --- a/ecs_composex/s3/s3_template.py +++ b/ecs_composex/s3/s3_template.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from troposphere import Ref, Sub, s3, AWS_NO_VALUE +from troposphere import Ref, Sub, s3, AWS_NO_VALUE, AWS_ACCOUNT_ID, AWS_REGION from ecs_composex.common import keyisset, keypresent, LOG from ecs_composex.s3 import metadata @@ -162,17 +162,26 @@ def define_access_control(properties): return properties["AccessControl"] -def define_accelerate_config(properties, settings): +def define_accelerate_config(properties, settings, bucket_name): """ Function to define AccelerateConfiguration :param properties: :param settings: + :param bucket_name: The name of the bucket. :return: """ - config = s3.AccelerateConfiguration( - AccelerationStatus=s3.s3_transfer_acceleration_status("Suspended") - ) + config = Ref(AWS_NO_VALUE) + if ( + isinstance(bucket_name, str) + and bucket_name.find(".") >= 0 + or keyisset("BucketName", properties) + and properties["BucketName"].find(".") > 0 + ): + LOG.warn( + "Your bucket name contains a `.` which is incompatible with Acceleration" + ) + return Ref(AWS_NO_VALUE) if keyisset("AccelerateConfiguration", properties): config = s3.AccelerateConfiguration( AccelerationStatus=s3.s3_transfer_acceleration_status("Suspended") @@ -189,6 +198,21 @@ def define_accelerate_config(properties, settings): def define_bucket_name(properties, settings): + """ + Function to automatically add Region and Account ID to the bucket name. + If set, will use a user-defined separator, else, `-` + + :param dict properties: + :param dict settings: + :return: The bucket name + :rtype: str + """ + separator = ( + settings["NameSeparator"] + if keyisset("NameSeparator", settings) + and isinstance(settings["NameSeparator"], str) + else r"-" + ) expand_region_key = "ExpandRegionToBucket" expand_account_id = "ExpandAccountIdToBucket" base_name = ( @@ -203,15 +227,15 @@ def define_bucket_name(properties, settings): if keyisset(expand_region_key, settings) and keyisset( expand_account_id, settings ): - return Sub(f"{base_name}.${{AWS::AccountId}}.${{AWS::Region}}") + return f"{base_name}{separator}${{{AWS_ACCOUNT_ID}}}{separator}${{{AWS_REGION}}}" elif keyisset(expand_region_key, settings) and not keyisset( expand_account_id, settings ): - return Sub(f"{base_name}.${{AWS::Region}}") + return f"{base_name}{separator}${{{AWS_REGION}}}" elif not keyisset(expand_region_key, settings) and keyisset( expand_account_id, settings ): - return Sub(f"{base_name}.${{AWS::AccountId}}") + return f"{base_name}{separator}${{{AWS_ACCOUNT_ID}}}" elif not keyisset(expand_account_id, settings) and not keyisset( expand_region_key, settings ): @@ -230,15 +254,24 @@ def define_bucket(bucket): :param ecs_composex.s3.s3_stack.Bucket bucket: :return: """ + bucket_name = define_bucket_name(bucket.properties, bucket.settings) + final_bucket_name = ( + Sub(bucket_name) + if isinstance(bucket_name, str) + and (bucket_name.find(AWS_REGION) >= 0 or bucket_name.find(AWS_ACCOUNT_ID) >= 0) + else bucket_name + ) + LOG.debug(bucket_name) + LOG.debug(final_bucket_name) props = { "AccelerateConfiguration": define_accelerate_config( - bucket.properties, bucket.settings + bucket.properties, bucket.settings, bucket_name ), "BucketEncryption": handle_bucket_encryption( bucket.properties, bucket.settings ), "AccessControl": define_access_control(bucket.properties), - "BucketName": define_bucket_name(bucket.properties, bucket.settings), + "BucketName": final_bucket_name, "ObjectLockEnabled": define_objects_locking(bucket.properties), "PublicAccessBlockConfiguration": define_public_block_access(bucket.properties), "VersioningConfiguration": define_bucket_versioning(bucket.properties), diff --git a/use-cases/s3/simple_s3_bucket.yml b/use-cases/s3/simple_s3_bucket.yml index 96c933e63..e6baec3a4 100644 --- a/use-cases/s3/simple_s3_bucket.yml +++ b/use-cases/s3/simple_s3_bucket.yml @@ -78,6 +78,7 @@ x-s3: Properties: BucketName: bucket-04 Settings: + NameSeparator: "." ExpandRegionToBucket: False ExpandAccountIdToBucket: False EnableEncryption: AES256