Updated SNS topics imports. Updated post_processing to manage list (#656

)

* Adding SNS with SQS subscription test

ecs_composex/common/__init__.py

@@ -44,20 +44,40 @@ def nxtpow2(x):
     return int(pow(2, ceil(log(x, 2))))
 
 
-def get_nested_property(top_object, property_path: str, separator: str = None):
+def get_nested_property(
+    top_object, property_path: str, separator: str = None, to_update: list = None
+):
     if separator is None:
         separator = r"."
     elif separator and not isinstance(separator, str):
         raise TypeError("Separator must be a string")
     top_property_split = property_path.split(separator, 1)
-    if len(top_property_split) == 1 and hasattr(top_object, top_property_split[0]):
-        return (
-            top_object,
-            top_property_split[0],
-            getattr(top_object, top_property_split[0]),
+    if to_update is None:
+        to_update: list = []
+    if (
+        len(top_property_split) == 1
+        and hasattr(top_object, top_property_split[0])
+        and not isinstance(top_object, list)
+    ):
+        to_update.append(
+            (
+                top_object,
+                top_property_split[0],
+                getattr(top_object, top_property_split[0]),
+            )
         )
+
+    if len(top_property_split) == 1 and isinstance(top_object, list):
+        for item in top_object:
+            get_nested_property(
+                item, top_property_split[0], separator=separator, to_update=to_update
+            )
+
     if len(top_property_split) > 1 and hasattr(top_object, top_property_split[0]):
         return get_nested_property(
-            getattr(top_object, top_property_split[0]), top_property_split[-1]
+            getattr(top_object, top_property_split[0]),
+            top_property_split[-1],
+            separator=separator,
+            to_update=to_update,
         )
-    return None, None, None
+    return to_update

ecs_composex/compose/x_resources/__init__.py

@@ -672,7 +672,6 @@ def add_parameter_to_family_stack(
     def add_attribute_to_another_stack(
         self, ext_stack, attribute: Parameter, settings: ComposeXSettings
     ):
-
         attr_id = self.attributes_outputs[attribute]
         if self.mappings and self.lookup:
             add_update_mapping(
@@ -691,38 +690,58 @@ def post_processing(self, settings: ComposeXSettings):
             LOG.debug("Not a new cluster or no post_processing_properties. Skipping")
             return
         LOG.info(f"Post processing {self.module.res_key}.{self.name}")
-        for _property in self.post_processing_properties:
-            cluster_property, property_name, value = get_nested_property(
+        for _property in getattr(self, "post_processing_properties", []):
+            properties_to_update: list[tuple] = get_nested_property(
                 self.cfn_resource, _property
             )
-            if not value or not isinstance(value, (str, list)):
-                continue
-            if (
-                isinstance(value, list)
-                and value
-                and isinstance(value[0], str)
-                and value[0].startswith(X_KEY)
-            ):
-                value = value[0]
-            if not value.startswith(X_KEY):
-                continue
-            resource, parameter = settings.get_resource_attribute(value)
-            if not resource or not parameter:
-                LOG.error(
-                    f"Failed to find resource/attribute for {property_name} with value {value}"
+            for _prop_to_update in properties_to_update:
+                aws_property_object, property_name, value = _prop_to_update
+                value = validate_input_value(aws_property_object, property_name, value)
+                if not value:
+                    continue
+                resource, parameter = settings.get_resource_attribute(value)
+                if not resource or not parameter:
+                    LOG.error(
+                        "%s.%s - Failed to find resource/attribute for %s with value %s",
+                        self.module.res_key,
+                        self.name,
+                        resource,
+                        value,
+                    )
+                    continue
+                res_param_id = resource.add_attribute_to_another_stack(
+                    self.stack, parameter, settings
                 )
-                continue
-            res_param_id = resource.add_attribute_to_another_stack(
-                self.stack, parameter, settings
-            )
-            if res_param_id is resource:
-                res_propery_value = Ref(resource.cfn_resource)
-            elif res_param_id is not resource and resource.cfn_resource:
-                res_propery_value = Ref(res_param_id["ImportParameter"])
-            else:
-                res_propery_value = res_param_id["ImportValue"]
-            setattr(cluster_property, property_name, res_propery_value)
-            LOG.info(
-                f"{self.module.res_key}.{self.name}"
-                f" - Successfully mapped {_property} to {resource.module.res_key}.{resource.name}",
-            )
+                if res_param_id is resource:
+                    res_propery_value = Ref(resource.cfn_resource)
+                elif res_param_id is not resource and resource.cfn_resource:
+                    res_propery_value = Ref(res_param_id["ImportParameter"])
+                else:
+                    res_propery_value = res_param_id["ImportValue"]
+                setattr(aws_property_object, property_name, res_propery_value)
+                LOG.info(
+                    "%s.%s - Successfully mapped %s to %s.%s",
+                    self.module.res_key,
+                    self.name,
+                    _property,
+                    resource.module.res_key,
+                    resource.name,
+                )
+
+
+def validate_input_value(aws_property_object, property_name, value) -> Union[None, str]:
+    """Validation that input for resource property update if valid"""
+    if (
+        (aws_property_object is None or property_name is None or value is None)
+        or (not value or not isinstance(value, (str, list)))
+        or (isinstance(value, str) and not value.startswith(X_KEY))
+    ):
+        return None
+    if (
+        isinstance(value, list)
+        and value
+        and isinstance(value[0], str)
+        and value[0].startswith(X_KEY)
+    ):
+        value = value[0]
+    return value

ecs_composex/ecs_composex.py

@@ -309,7 +309,9 @@ def generate_full_template(settings: ComposeXSettings):
     set_all_mappings_to_root_stack(settings.root_stack, settings)
 
     for resource in settings.x_resources:
-        if hasattr(resource, "post_processing"):
+        if hasattr(resource, "post_processing") and hasattr(
+            resource, "post_processing_properties"
+        ):
             resource.post_processing(settings)
 
     settings.mod_manager.modules.clear()

ecs_composex/sns/sns_stack.py

@@ -14,14 +14,9 @@
 from ecs_composex.common.stacks import ComposeXStack
 from ecs_composex.common.troposphere_tools import build_template
 from ecs_composex.compose.x_resources.api_x_resources import ApiXResource
-from ecs_composex.compose.x_resources.helpers import (
-    set_lookup_resources,
-    set_new_resources,
-    set_resources,
-)
 from ecs_composex.sns.sns_helpers import create_sns_mappings
 from ecs_composex.sns.sns_params import TOPIC_ARN, TOPIC_NAME
-from ecs_composex.sns.sns_templates import generate_sns_templates
+from ecs_composex.sns.sns_templates import import_sns_topics_to_template
 
 
 class Topic(ApiXResource):
@@ -40,6 +35,7 @@ def __init__(
         self.arn_parameter = TOPIC_ARN
         self.ref_parameter = TOPIC_ARN
         self.support_defaults = True
+        self.post_processing_properties = ["Subscription.Endpoint"]
 
     def init_outputs(self):
         self.output_properties = {
@@ -67,10 +63,8 @@ def __init__(
         if not module.new_resources:
             self.is_void = True
         else:
-            template = build_template(
-                f"{module.res_key} - Compose-X Generated template"
-            )
-            generate_sns_templates(settings, module.new_resources, self, template)
+            template = build_template(f"{module.res_key} - stack")
+            import_sns_topics_to_template(module.new_resources, template)
             super().__init__(module.mapping_key, stack_template=template, **kwargs)
         for resource in module.resources_list:
             resource.stack = self

ecs_composex/sns/sns_templates.py

@@ -5,119 +5,38 @@
 Module to add topics and subscriptions to the SNS stack
 """
 
-from compose_x_common.compose_x_common import keyisset
-from troposphere.sns import Topic
+from __future__ import annotations
 
-from ecs_composex.common.logging import LOG
-from ecs_composex.common.troposphere_tools import add_outputs
-from ecs_composex.sns import metadata
-
-TOPICS_KEY = "Topics"
-SUBSCRIPTIONS_KEY = "Subscription"
-TOPICS_STACK_NAME = "topics"
-ENDPOINT_KEY = "Endpoint"
-PROTOCOL_KEY = "Protocol"
-
-
-def define_topic_subscriptions(subscriptions, content):
-    """
-    Function to define an SNS topic subscriptions
+from typing import TYPE_CHECKING
 
-    :param list subscriptions: list of subscriptions as defined in the docker compose file
-    :param dict content: docker compose file content
-    :return:
-    """
-    required_keys = [ENDPOINT_KEY, PROTOCOL_KEY]
-    subscriptions_objs = []
-    for sub in subscriptions:
-        LOG.debug(sub)
-        if not all(key in sub for key in required_keys):
-            raise AttributeError(
-                "Required attributes for Subscription are",
-                required_keys,
-                "Provided",
-                sub.keys(),
-            )
-        if keyisset(PROTOCOL_KEY, sub) and (
-            sub[PROTOCOL_KEY] == "sqs" or sub[PROTOCOL_KEY] == "SQS"
-        ):
-            pass
-        else:
-            subscriptions_objs.append(sub)
-    return subscriptions_objs
-
-
-def define_topic(topic, content):
-    """
-    Function that builds the SNS topic template from cli.Dockerfile Properties
-
-    :param topic: The topic and its definition
-    :type topic: ecs_composex.sns.sns_stack.Topic
-    """
-    topic.cfn_resource = Topic(topic.logical_name, Metadata=metadata)
-    if keyisset(SUBSCRIPTIONS_KEY, topic.properties):
-        subscriptions = define_topic_subscriptions(
-            topic.properties[SUBSCRIPTIONS_KEY], content
-        )
-        setattr(topic.cfn_resource, "Subscription", subscriptions)
+if TYPE_CHECKING:
+    from troposphere import Template
+    from ecs_composex.sns.sns_stack import Topic
 
-        for key in topic.properties.keys():
-            if type(topic.properties[key]) != list:
-                setattr(topic.cfn_resource, key, topic.properties[key])
+from troposphere.sns import Topic as CfnTopic
 
+from ecs_composex.common.troposphere_tools import add_outputs
+from ecs_composex.resources_import import import_record_properties
+from ecs_composex.sns import metadata
 
-def add_topics_to_template(template, topics, content):
-    """
-    Function to interate over the topics and add them to the CFN Template
 
-    :param troposphere.Template template:
-    :param dict topics:
-    :param dict content: Content of the compose file
-    """
+def add_topics_to_template(template, topics):
+    """Function to interate over the topics and add them to the CFN Template"""
     for topic in topics:
-        define_topic(topic, content)
+        topic_props = import_record_properties(topic.properties, CfnTopic)
+        topic.cfn_resource = CfnTopic(
+            topic.logical_name, Metadata=metadata, **topic_props
+        )
         topic.init_outputs()
         topic.generate_outputs()
         template.add_resource(topic.cfn_resource)
         add_outputs(template, topic.outputs)
 
 
-def add_sns_topics(root_template, new_topics, content):
-    """
-    Function to add SNS topics to the root template
-
-    :param troposphere.Template root_template:
-    :param new_topics:
-    :param dict content:
-    :return:
-    """
-    add_topics_to_template(root_template, new_topics, content)
-
-
-def define_resources(res_content):
-    """
-    Function to determine how many resources are going to be created.
-    :return:
-    """
-    res_count = 0
-    if keyisset(TOPICS_KEY, res_content):
-        for topic in res_content[TOPICS_KEY]:
-            res_count += 1
-            if keyisset("Subscription", topic):
-                res_count += len(topic["Subscription"])
-    if keyisset(SUBSCRIPTIONS_KEY, res_content):
-        res_count += len(res_content[SUBSCRIPTIONS_KEY])
-    return res_count
-
-
-def generate_sns_templates(settings, new_topics, xstack, root_template):
-    """
-    Entrypoint function to generate the SNS topics templates
-
-    :param settings:
-    :type settings: ecs_composex.common.settings.ComposeXSettings
-    :return:
-    """
-    if new_topics:
-        add_sns_topics(root_template, new_topics, settings.compose_content)
+def import_sns_topics_to_template(
+    new_topics: list[Topic],
+    root_template: Template,
+):
+    """Entrypoint function to generate the SNS topics templates"""
+    add_topics_to_template(root_template, new_topics)
     return root_template

tests/features/features/sns.feature

@@ -10,6 +10,7 @@ Feature: ecs_composex.sns
 
 
         Examples:
-            | file_path                   | override_file                       |
-            | use-cases/blog.features.yml | use-cases/sns/simple_sns.yml        |
-            | use-cases/blog.features.yml | use-cases/sns/create_and_lookup.yml |
+            | file_path                   | override_file                                |
+            | use-cases/blog.features.yml | use-cases/sns/simple_sns.yml                 |
+            | use-cases/blog.features.yml | use-cases/sns/create_and_lookup.yml          |
+            | use-cases/blog.features.yml | use-cases/sns/sns_with_sqs_subscription.yaml |

use-cases/sns/sns_with_sqs_subscription.yaml

@@ -0,0 +1,30 @@
+x-sns:
+  abcd:
+    Properties: {}
+    Services:
+      app01:
+        Access: Publish
+      youtoo:
+        Access: Publish
+
+
+  someothertopic:
+    Properties:
+      Subscription:
+        - Endpoint: x-sqs::queueA::Arn
+          Protocol: "sqs"
+        - Endpoint: x-sqs::queueC::Arn
+          Protocol: "sqs"
+      TopicName: "SampleTopic"
+
+
+x-sqs:
+  queueA: {}
+  queueC:
+    Properties:
+      FifoQueue: true
+    Services:
+      rproxy:
+        Access: RWMessages
+      youtoo:
+        Access: RWMessages