diff --git a/ecs_composex/common/__init__.py b/ecs_composex/common/__init__.py index 730abacd..84c584a1 100644 --- a/ecs_composex/common/__init__.py +++ b/ecs_composex/common/__init__.py @@ -44,20 +44,40 @@ def nxtpow2(x): return int(pow(2, ceil(log(x, 2)))) -def get_nested_property(top_object, property_path: str, separator: str = None): +def get_nested_property( + top_object, property_path: str, separator: str = None, to_update: list = None +): if separator is None: separator = r"." elif separator and not isinstance(separator, str): raise TypeError("Separator must be a string") top_property_split = property_path.split(separator, 1) - if len(top_property_split) == 1 and hasattr(top_object, top_property_split[0]): - return ( - top_object, - top_property_split[0], - getattr(top_object, top_property_split[0]), + if to_update is None: + to_update: list = [] + if ( + len(top_property_split) == 1 + and hasattr(top_object, top_property_split[0]) + and not isinstance(top_object, list) + ): + to_update.append( + ( + top_object, + top_property_split[0], + getattr(top_object, top_property_split[0]), + ) ) + + if len(top_property_split) == 1 and isinstance(top_object, list): + for item in top_object: + get_nested_property( + item, top_property_split[0], separator=separator, to_update=to_update + ) + if len(top_property_split) > 1 and hasattr(top_object, top_property_split[0]): return get_nested_property( - getattr(top_object, top_property_split[0]), top_property_split[-1] + getattr(top_object, top_property_split[0]), + top_property_split[-1], + separator=separator, + to_update=to_update, ) - return None, None, None + return to_update diff --git a/ecs_composex/compose/x_resources/__init__.py b/ecs_composex/compose/x_resources/__init__.py index d76da8d1..d6b6e3db 100644 --- a/ecs_composex/compose/x_resources/__init__.py +++ b/ecs_composex/compose/x_resources/__init__.py @@ -672,7 +672,6 @@ def add_parameter_to_family_stack( def add_attribute_to_another_stack( self, ext_stack, attribute: Parameter, settings: ComposeXSettings ): - attr_id = self.attributes_outputs[attribute] if self.mappings and self.lookup: add_update_mapping( @@ -691,38 +690,58 @@ def post_processing(self, settings: ComposeXSettings): LOG.debug("Not a new cluster or no post_processing_properties. Skipping") return LOG.info(f"Post processing {self.module.res_key}.{self.name}") - for _property in self.post_processing_properties: - cluster_property, property_name, value = get_nested_property( + for _property in getattr(self, "post_processing_properties", []): + properties_to_update: list[tuple] = get_nested_property( self.cfn_resource, _property ) - if not value or not isinstance(value, (str, list)): - continue - if ( - isinstance(value, list) - and value - and isinstance(value[0], str) - and value[0].startswith(X_KEY) - ): - value = value[0] - if not value.startswith(X_KEY): - continue - resource, parameter = settings.get_resource_attribute(value) - if not resource or not parameter: - LOG.error( - f"Failed to find resource/attribute for {property_name} with value {value}" + for _prop_to_update in properties_to_update: + aws_property_object, property_name, value = _prop_to_update + value = validate_input_value(aws_property_object, property_name, value) + if not value: + continue + resource, parameter = settings.get_resource_attribute(value) + if not resource or not parameter: + LOG.error( + "%s.%s - Failed to find resource/attribute for %s with value %s", + self.module.res_key, + self.name, + resource, + value, + ) + continue + res_param_id = resource.add_attribute_to_another_stack( + self.stack, parameter, settings ) - continue - res_param_id = resource.add_attribute_to_another_stack( - self.stack, parameter, settings - ) - if res_param_id is resource: - res_propery_value = Ref(resource.cfn_resource) - elif res_param_id is not resource and resource.cfn_resource: - res_propery_value = Ref(res_param_id["ImportParameter"]) - else: - res_propery_value = res_param_id["ImportValue"] - setattr(cluster_property, property_name, res_propery_value) - LOG.info( - f"{self.module.res_key}.{self.name}" - f" - Successfully mapped {_property} to {resource.module.res_key}.{resource.name}", - ) + if res_param_id is resource: + res_propery_value = Ref(resource.cfn_resource) + elif res_param_id is not resource and resource.cfn_resource: + res_propery_value = Ref(res_param_id["ImportParameter"]) + else: + res_propery_value = res_param_id["ImportValue"] + setattr(aws_property_object, property_name, res_propery_value) + LOG.info( + "%s.%s - Successfully mapped %s to %s.%s", + self.module.res_key, + self.name, + _property, + resource.module.res_key, + resource.name, + ) + + +def validate_input_value(aws_property_object, property_name, value) -> Union[None, str]: + """Validation that input for resource property update if valid""" + if ( + (aws_property_object is None or property_name is None or value is None) + or (not value or not isinstance(value, (str, list))) + or (isinstance(value, str) and not value.startswith(X_KEY)) + ): + return None + if ( + isinstance(value, list) + and value + and isinstance(value[0], str) + and value[0].startswith(X_KEY) + ): + value = value[0] + return value diff --git a/ecs_composex/ecs_composex.py b/ecs_composex/ecs_composex.py index 7c463095..c2c95515 100644 --- a/ecs_composex/ecs_composex.py +++ b/ecs_composex/ecs_composex.py @@ -309,7 +309,9 @@ def generate_full_template(settings: ComposeXSettings): set_all_mappings_to_root_stack(settings.root_stack, settings) for resource in settings.x_resources: - if hasattr(resource, "post_processing"): + if hasattr(resource, "post_processing") and hasattr( + resource, "post_processing_properties" + ): resource.post_processing(settings) settings.mod_manager.modules.clear() diff --git a/ecs_composex/sns/sns_stack.py b/ecs_composex/sns/sns_stack.py index 3cea1722..50923426 100644 --- a/ecs_composex/sns/sns_stack.py +++ b/ecs_composex/sns/sns_stack.py @@ -14,14 +14,9 @@ from ecs_composex.common.stacks import ComposeXStack from ecs_composex.common.troposphere_tools import build_template from ecs_composex.compose.x_resources.api_x_resources import ApiXResource -from ecs_composex.compose.x_resources.helpers import ( - set_lookup_resources, - set_new_resources, - set_resources, -) from ecs_composex.sns.sns_helpers import create_sns_mappings from ecs_composex.sns.sns_params import TOPIC_ARN, TOPIC_NAME -from ecs_composex.sns.sns_templates import generate_sns_templates +from ecs_composex.sns.sns_templates import import_sns_topics_to_template class Topic(ApiXResource): @@ -40,6 +35,7 @@ def __init__( self.arn_parameter = TOPIC_ARN self.ref_parameter = TOPIC_ARN self.support_defaults = True + self.post_processing_properties = ["Subscription.Endpoint"] def init_outputs(self): self.output_properties = { @@ -67,10 +63,8 @@ def __init__( if not module.new_resources: self.is_void = True else: - template = build_template( - f"{module.res_key} - Compose-X Generated template" - ) - generate_sns_templates(settings, module.new_resources, self, template) + template = build_template(f"{module.res_key} - stack") + import_sns_topics_to_template(module.new_resources, template) super().__init__(module.mapping_key, stack_template=template, **kwargs) for resource in module.resources_list: resource.stack = self diff --git a/ecs_composex/sns/sns_templates.py b/ecs_composex/sns/sns_templates.py index a605db77..cd08b0e1 100644 --- a/ecs_composex/sns/sns_templates.py +++ b/ecs_composex/sns/sns_templates.py @@ -5,119 +5,38 @@ Module to add topics and subscriptions to the SNS stack """ -from compose_x_common.compose_x_common import keyisset -from troposphere.sns import Topic +from __future__ import annotations -from ecs_composex.common.logging import LOG -from ecs_composex.common.troposphere_tools import add_outputs -from ecs_composex.sns import metadata - -TOPICS_KEY = "Topics" -SUBSCRIPTIONS_KEY = "Subscription" -TOPICS_STACK_NAME = "topics" -ENDPOINT_KEY = "Endpoint" -PROTOCOL_KEY = "Protocol" - - -def define_topic_subscriptions(subscriptions, content): - """ - Function to define an SNS topic subscriptions +from typing import TYPE_CHECKING - :param list subscriptions: list of subscriptions as defined in the docker compose file - :param dict content: docker compose file content - :return: - """ - required_keys = [ENDPOINT_KEY, PROTOCOL_KEY] - subscriptions_objs = [] - for sub in subscriptions: - LOG.debug(sub) - if not all(key in sub for key in required_keys): - raise AttributeError( - "Required attributes for Subscription are", - required_keys, - "Provided", - sub.keys(), - ) - if keyisset(PROTOCOL_KEY, sub) and ( - sub[PROTOCOL_KEY] == "sqs" or sub[PROTOCOL_KEY] == "SQS" - ): - pass - else: - subscriptions_objs.append(sub) - return subscriptions_objs - - -def define_topic(topic, content): - """ - Function that builds the SNS topic template from cli.Dockerfile Properties - - :param topic: The topic and its definition - :type topic: ecs_composex.sns.sns_stack.Topic - """ - topic.cfn_resource = Topic(topic.logical_name, Metadata=metadata) - if keyisset(SUBSCRIPTIONS_KEY, topic.properties): - subscriptions = define_topic_subscriptions( - topic.properties[SUBSCRIPTIONS_KEY], content - ) - setattr(topic.cfn_resource, "Subscription", subscriptions) +if TYPE_CHECKING: + from troposphere import Template + from ecs_composex.sns.sns_stack import Topic - for key in topic.properties.keys(): - if type(topic.properties[key]) != list: - setattr(topic.cfn_resource, key, topic.properties[key]) +from troposphere.sns import Topic as CfnTopic +from ecs_composex.common.troposphere_tools import add_outputs +from ecs_composex.resources_import import import_record_properties +from ecs_composex.sns import metadata -def add_topics_to_template(template, topics, content): - """ - Function to interate over the topics and add them to the CFN Template - :param troposphere.Template template: - :param dict topics: - :param dict content: Content of the compose file - """ +def add_topics_to_template(template, topics): + """Function to interate over the topics and add them to the CFN Template""" for topic in topics: - define_topic(topic, content) + topic_props = import_record_properties(topic.properties, CfnTopic) + topic.cfn_resource = CfnTopic( + topic.logical_name, Metadata=metadata, **topic_props + ) topic.init_outputs() topic.generate_outputs() template.add_resource(topic.cfn_resource) add_outputs(template, topic.outputs) -def add_sns_topics(root_template, new_topics, content): - """ - Function to add SNS topics to the root template - - :param troposphere.Template root_template: - :param new_topics: - :param dict content: - :return: - """ - add_topics_to_template(root_template, new_topics, content) - - -def define_resources(res_content): - """ - Function to determine how many resources are going to be created. - :return: - """ - res_count = 0 - if keyisset(TOPICS_KEY, res_content): - for topic in res_content[TOPICS_KEY]: - res_count += 1 - if keyisset("Subscription", topic): - res_count += len(topic["Subscription"]) - if keyisset(SUBSCRIPTIONS_KEY, res_content): - res_count += len(res_content[SUBSCRIPTIONS_KEY]) - return res_count - - -def generate_sns_templates(settings, new_topics, xstack, root_template): - """ - Entrypoint function to generate the SNS topics templates - - :param settings: - :type settings: ecs_composex.common.settings.ComposeXSettings - :return: - """ - if new_topics: - add_sns_topics(root_template, new_topics, settings.compose_content) +def import_sns_topics_to_template( + new_topics: list[Topic], + root_template: Template, +): + """Entrypoint function to generate the SNS topics templates""" + add_topics_to_template(root_template, new_topics) return root_template diff --git a/tests/features/features/sns.feature b/tests/features/features/sns.feature index b97430d4..9d109e10 100644 --- a/tests/features/features/sns.feature +++ b/tests/features/features/sns.feature @@ -10,6 +10,7 @@ Feature: ecs_composex.sns Examples: - | file_path | override_file | - | use-cases/blog.features.yml | use-cases/sns/simple_sns.yml | - | use-cases/blog.features.yml | use-cases/sns/create_and_lookup.yml | + | file_path | override_file | + | use-cases/blog.features.yml | use-cases/sns/simple_sns.yml | + | use-cases/blog.features.yml | use-cases/sns/create_and_lookup.yml | + | use-cases/blog.features.yml | use-cases/sns/sns_with_sqs_subscription.yaml | diff --git a/use-cases/sns/sns_with_sqs_subscription.yaml b/use-cases/sns/sns_with_sqs_subscription.yaml new file mode 100644 index 00000000..41da2ee5 --- /dev/null +++ b/use-cases/sns/sns_with_sqs_subscription.yaml @@ -0,0 +1,30 @@ +x-sns: + abcd: + Properties: {} + Services: + app01: + Access: Publish + youtoo: + Access: Publish + + + someothertopic: + Properties: + Subscription: + - Endpoint: x-sqs::queueA::Arn + Protocol: "sqs" + - Endpoint: x-sqs::queueC::Arn + Protocol: "sqs" + TopicName: "SampleTopic" + + +x-sqs: + queueA: {} + queueC: + Properties: + FifoQueue: true + Services: + rproxy: + Access: RWMessages + youtoo: + Access: RWMessages