Refactor to classes for XResources and Compose resources (#219)

* Mapped volumes <> services
* Mapped secrets <> services
* Mapping x-resources to services/families
* Working and easy implementation for SQS
* Working rework of S3
* Re-implemented scaling
* Handled lookup import for SQS
* Added some x-cluster use-cases
* Using tag API and new functions for DynDB
* Working SNS lookup
* Using Route53 zone for public in priority over CloudMap
* Factorized simple IAM permissions access and Lookup for KMS Keys

Makefile

@@ -69,7 +69,7 @@ test-all: ## run tests on every Python version with tox
 
 coverage: ## check code coverage quickly with the default Python
 	coverage run --source ecs_composex -m behave --junit || exit 0
-	coverage run --source ecs_composex -a -m pytest pytests -vv -x
+	coverage run --source ecs_composex -a -m pytest pytests -vv -x || exit 0
 	coverage report -m
 	coverage html
 	$(BROWSER) htmlcov/index.html

ecs_composex/appmesh/appmesh_mesh.py

@@ -54,9 +54,7 @@ class Mesh(object):
     services_key = "services"
     required_keys = [nodes_key, routers_key, services_key]
 
-    def __init__(
-        self, mesh_definition, services_families, services_root_stack, settings
-    ):
+    def __init__(self, mesh_definition, services_root_stack, settings):
         """
         Method to initialize the Mesh
 
@@ -132,11 +130,11 @@ def __init__(
         for key in self.required_keys:
             if key not in self.mesh_settings.keys():
                 raise KeyError(f"Key {key} is missing. Required {self.required_keys}")
-        self.define_nodes(services_families, services_root_stack)
+        self.define_nodes(settings, services_root_stack)
         self.define_routes_and_routers()
         self.define_virtual_services()
 
-    def define_nodes(self, services_families, services_root_stack):
+    def define_nodes(self, settings, services_root_stack):
         """
         Method to compile the nodes for the Mesh.
 
@@ -150,24 +148,35 @@ def define_nodes(self, services_families, services_root_stack):
                 raise AttributeError(
                     f"Nodes must have set {nodes_keys}. Got", node.keys()
                 )
-            service_family = get_service_family_name(services_families, node["name"])
-            LOG.debug(service_family)
-            if service_family not in services_root_stack.stack_template.resources:
-                raise AttributeError(
-                    f"Node defined {service_family} is not defined in services stack",
-                    services_root_stack.stack_template.resources,
+            service_families = [
+                settings.families[name]
+                for name in settings.families
+                if settings.families[name].name == node["name"]
+            ]
+            if len(service_families) > 1:
+                raise LookupError(
+                    "More than one family matched for the node.",
+                    service_families,
+                    node["name"],
+                )
+            elif not service_families:
+                raise LookupError(
+                    "No family could be matched for the given node",
+                    settings.families,
+                    node["name"],
                 )
             LOG.debug(node)
-            self.nodes[service_family] = MeshNode(
-                services_root_stack.stack_template.resources[service_family],
+            family = service_families[0]
+            self.nodes[family.logical_name] = MeshNode(
+                family,
                 node["protocol"],
                 node["backends"] if keyisset("backends", node) else None,
             )
-            self.nodes[service_family].get_node_param = GetAtt(
-                self.nodes[service_family].param_name, "Outputs.VirtualNode"
+            self.nodes[family.logical_name].get_node_param = GetAtt(
+                self.nodes[family.logical_name].param_name, "Outputs.VirtualNode"
             )
-            self.nodes[service_family].get_sg_param = GetAtt(
-                self.nodes[service_family].param_name,
+            self.nodes[family.logical_name].get_sg_param = GetAtt(
+                self.nodes[family.logical_name].param_name,
                 f"Outputs.{ecs_params.SERVICE_GROUP_ID_T}",
             )
 

ecs_composex/appmesh/appmesh_node.py

@@ -35,7 +35,7 @@
 from ecs_composex.common.outputs import ComposeXOutput
 from ecs_composex.dns.dns_params import PRIVATE_DNS_ZONE_NAME
 from ecs_composex.ecs import ecs_params
-from ecs_composex.ecs.ecs_container_config import extend_container_envvars
+from ecs_composex.common.compose_services import extend_container_envvars
 
 
 class MeshNode(object):
@@ -45,24 +45,25 @@ class MeshNode(object):
 
     weight = 1
 
-    def __init__(self, service_stack, protocol, backends=None):
+    def __init__(self, family, protocol, backends=None):
         """
-
-        :param ecs_composex.ServiceStack service_stack: the service template
+        Creates the AppMesh VirtualNode pointing to the family service
         """
         self.node = None
-        self.param_name = service_stack.title
+        self.param_name = family.logical_name
         self.get_node_param = None
         self.get_sg_param = None
         self.backends = [] if backends is None else backends
-        self.stack = service_stack
+        self.stack = family.stack
+        self.template = family.template
+        self.service_config = family.service_config
         self.protocol = protocol
         self.mappings = {}
         self.port_mappings = []
         self.set_port_mappings()
         self.set_listeners_port_mappings()
         self.extend_service_stack()
-        self.add_envoy_container_definition()
+        self.add_envoy_container_definition(family)
         self.extend_task_policy()
 
     def set_port_mappings(self):
@@ -71,7 +72,7 @@ def set_port_mappings(self):
         """
         target = "target"
         published = "published"
-        for port in self.stack.config.ports:
+        for port in self.service_config.network.ports:
             if port[target] not in self.mappings.keys():
                 self.mappings[port[target]] = {port[published]: port}
             elif (
@@ -86,7 +87,7 @@ def set_listeners_port_mappings(self):
         """
         self.port_mappings = [
             appmesh.PortMapping(Port=port["published"], Protocol=self.protocol)
-            for port in self.stack.config.ports
+            for port in self.service_config.network.ports
         ]
 
     def extend_service_stack(self):
@@ -144,12 +145,12 @@ def set_node_weight(self, weight):
         """
         self.weight = weight
 
-    def add_envoy_container_definition(self):
+    def add_envoy_container_definition(self, family):
         """
         Method to expand the containers configuration and add the Envoy SideCar.
         """
         envoy_container_name = "envoy"
-        task = self.stack.service.task
+        task = family.task_definition
         envoy_port_mapping = [
             PortMapping(ContainerPort=15000, HostPort=15000),
             PortMapping(ContainerPort=15001, HostPort=15001),
@@ -163,7 +164,7 @@ def add_envoy_container_definition(self):
             ),
             Environment(
                 Name="ENABLE_ENVOY_XRAY_TRACING",
-                Value="1" if task.family_config.use_xray else "0",
+                Value="1" if family.use_xray else "0",
             ),
             Environment(Name="ENABLE_ENVOY_STATS_TAGS", Value="1"),
         ]
@@ -175,7 +176,7 @@ def add_envoy_container_definition(self):
                 "awslogs-stream-prefix": envoy_container_name,
             },
         )
-        self.stack.stack_template.add_parameter(appmesh_params.ENVOY_IMAGE_URL)
+        family.template.add_parameter(appmesh_params.ENVOY_IMAGE_URL)
         envoy_container = ContainerDefinition(
             Image=Ref(appmesh_params.ENVOY_IMAGE_URL),
             Name=envoy_container_name,
@@ -219,10 +220,9 @@ def add_envoy_container_definition(self):
                 ),
             ],
         )
-        task.containers.append(envoy_container)
-        setattr(task.definition, "ProxyConfiguration", proxy_config)
-        task.set_task_compute_parameter()
-        self.stack.Parameters.update(task.stack_parameters)
+        task.ContainerDefinitions.append(envoy_container)
+        setattr(family.task_definition, "ProxyConfiguration", proxy_config)
+        family.refresh()
 
     def extend_task_policy(self):
         """

ecs_composex/secrets/SYNTAX.rst → ecs_composex/common/SYNTAX.rst

@@ -33,8 +33,8 @@ To help make it as easy in AWS, simply set `external=True` and a few other setti
             - EcsExecutionRole
             - EcsTaskRole
 
-x-secret
---------
+x-secrets
+---------
 
 Name
 ^^^^

ecs_composex/common/__init__.py

@@ -42,7 +42,7 @@
 
 DATE = dt.utcnow().isoformat()
 FILE_PREFIX = f'{dt.utcnow().strftime("%Y/%m/%d/%H%M")}/{str(uuid4().hex)[:6]}'
-NONALPHANUM = re.compile(r"([^a-zA-Z0-9])")
+NONALPHANUM = re.compile(r"([^a-zA-Z0-9]+)")
 
 EXIT_CODES = {"MODULE_NOT_FOUND": 8, "MISSING_RESOURCE_DEFINITION": 9}
 

ecs_composex/common/aws.py

@@ -51,6 +51,7 @@ def get_resources_from_tags(session, aws_resource_search, search_tags):
     :return:
     """
     try:
+
         client = session.client("resourcegroupstaggingapi")
         resources_r = client.get_resources(
             ResourceTypeFilters=[aws_resource_search], TagFilters=search_tags
@@ -111,7 +112,9 @@ def handle_search_results(arns, name, res_types, aws_resource_search):
     """
     if not arns:
         raise LookupError(
-            "No resources were found with the provided tags and information"
+            "No resources were found with the provided tags and information",
+            name,
+            aws_resource_search,
         )
     if arns and isinstance(name, str):
         return handle_multi_results(
@@ -169,6 +172,7 @@ def find_aws_resource_arn_from_tags_api(info, session, aws_resource_search, type
     name = info["Name"] if keyisset("Name", info) else None
 
     resources_r = get_resources_from_tags(session, aws_resource_search, search_tags)
+    LOG.debug(search_tags)
     arns = [i["ResourceARN"] for i in resources_r["ResourceTagMappingList"]]
     return handle_search_results(arns, name, res_types, aws_resource_search)
 

ecs_composex/common/cfn_params.py

@@ -32,6 +32,7 @@
 MESH_TITLE = "RootMesh"
 PRIVATE_MAP_TITLE = "CloudMapVpcNamespace"
 PUBLIC_MAP_TITLE = "CloudMapPublicNamespace"
+PUBLIC_ZONE_TITLE = "Route53PublicZone"
 
 ROOT_STACK_NAME_T = "RootStackName"
 ROOT_STACK_NAME = Parameter(