ECS Scheduled tasks support (#280)

* Scheduled Tasks tested and working * Stronger IAM policies * Added events to index
compose-x · Nov 26, 2020 · ba4ed5c · ba4ed5c
1 parent 42c7027
commit ba4ed5c
Show file tree

Hide file tree

Showing 20 changed files with 628 additions and 33 deletions.
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: clean clean-test clean-pyc clean-build docs help lint conform release-test release
+.PHONY: clean clean-test clean-pyc clean-build docs help lint conform release-test release codebuild coverage
 .DEFAULT_GOAL := help
 
 define BROWSER_PYSCRIPT
@@ -71,9 +71,16 @@ coverage: ## check code coverage quickly with the default Python
 	coverage run --source ecs_composex -m behave --junit || exit 0
 	coverage run --source ecs_composex -a -m pytest pytests -vv -x || exit 0
 	coverage report -m
+	coverage xml -o coverage/coverage.xml
 	coverage html
 	$(BROWSER) htmlcov/index.html
 
+codebuild: ## check code coverage quickly with the default Python
+	coverage run --source ecs_composex -m behave --junit || exit 0
+	coverage run --source ecs_composex -a -m pytest pytests -vv -x || exit 0
+	coverage report -m
+	coverage xml -o coverage/coverage.xml
+
 docs: clean-c9 ## generate Sphinx HTML documentation, including API docs
 	rm -f docs/ecs_composex.rst
 	rm -f docs/modules.rst

diff --git a/README.rst b/README.rst
@@ -6,7 +6,7 @@ ECS ComposeX
 
 |CODE_STYLE| |TDD| |BDD|
 
-|CODECOV| |QUALITY|
+|QUALITY|
 
 |BUILD|
 
@@ -282,11 +282,6 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
 .. |PYPI_VERSION| image:: https://img.shields.io/pypi/v/ecs_composex.svg
         :target: https://pypi.python.org/pypi/ecs_composex
 
-
-.. |CODECOV| image:: https://img.shields.io/codecov/c/github/lambda-my-aws/ecs_composex?color=black&style=flat-square
-    :alt: Codecov
-    :target: https://codecov.io/gh/lambda-my-aws/ecs_composex
-
 .. |PYPI_DL| image:: https://img.shields.io/pypi/dm/ecs_composex
     :alt: PyPI - Downloads
     :target: https://pypi.python.org/pypi/ecs_composex

diff --git a/buildspec.yml b/buildspec.yml
@@ -20,20 +20,18 @@ phases:
   build:
     commands:
       - echo "Executing behave"
-      - if ! [ -d reports ]; then mkdir reports ; fi
-      - make coverage
-      - curl -s https://codecov.io/bash | bash
-
-artifacts:
-  files:
-    - '*.yml'
-    - '*.json'
-  base-directory: outputs
-  name: outputs
+      - if ! [ -d reports ]; then mkdir reports ; else rm -rf reports/* ; fi
+      - if ! [ -d coverage ]; then mkdir coverage ; else rm -rf coverage/* ; fi
+      - make codebuild
 
 reports:
-  composexbehave:
+  composex-tests:
     files:
       - "*.xml"
     base-directory: reports
     file-format: JunitXml
+  composex-coverage:
+    files:
+      - "coverage.xml"
+    base-directory: coverage
+    file-format: COBERTURAXML
diff --git a/codecov.yml b/codecov.yml
diff --git a/docs/modules_syntax.rst b/docs/modules_syntax.rst
@@ -35,6 +35,7 @@
     syntax/composex/acm
     syntax/composex/kms
     syntax/composex/vpc
+    syntax/composex/events
     syntax/composex/ecs_cluster
     syntax/composex/dns
     syntax/composex/compute
diff --git a/docs/readme.rst b/docs/readme.rst
@@ -6,7 +6,7 @@ ECS ComposeX
 
 |CODE_STYLE| |TDD| |BDD|
 
-|CODECOV| |QUALITY|
+|QUALITY|
 
 |BUILD|
 
@@ -217,11 +217,6 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
 .. |PYPI_VERSION| image:: https://img.shields.io/pypi/v/ecs_composex.svg
         :target: https://pypi.python.org/pypi/ecs_composex
 
-
-.. |CODECOV| image:: https://img.shields.io/codecov/c/github/lambda-my-aws/ecs_composex?color=black&style=flat-square
-    :alt: Codecov
-    :target: https://codecov.io/gh/lambda-my-aws/ecs_composex
-
 .. |PYPI_DL| image:: https://img.shields.io/pypi/dm/ecs_composex
     :alt: PyPI - Downloads
     :target: https://pypi.python.org/pypi/ecs_composex

diff --git a/docs/syntax/composex/events.rst b/docs/syntax/composex/events.rst
@@ -0,0 +1,73 @@
+.. _events_syntax_reference:
+
+==========
+x-events
+==========
+
+This extension allows you to define an AWS EventBride rule to stop start services at specific times
+of the day or based on specific events.
+
+Properties
+==========
+
+You can find all the properties on the `AWS CFN Events Rules definitions`_.
+
+.. note::
+
+    You do not need to define Targets to point to the services defined in docker-compose. Refer to `Services`_ for that.
+
+MacroParameters
+================
+
+No specific parameters at this time!
+
+
+Settings
+========
+
+No specific settings at this time!
+
+Services
+========
+
+There we define the tasks we want to deploy at specific times or events.
+
+.. code-block:: yaml
+    :caption: Services syntax for rules
+
+    name: service_name
+    TaskCount: <N>
+    DeleteDefaultService: True/False (default. False)
+
+name
+""""
+
+Here we want to define the name of the **family** we want to use for trigger. If the service is not defined as part of a
+specific family, you can use the service name itself.
+
+.. seealso::
+
+    .. :ref:`composex_deploy_extension`
+
+*Required: Yes.*
+
+TaskCount
+"""""""""
+
+Same property as for ECS Parameters of the `Task Rule target definition`_ itself, this allows you to set a specific number
+of tasks.
+
+*Required: Yes.*
+
+.. hint::
+
+    Not using deploy/replicas on purpose, because of the `DeleteDefaultService`_ option
+
+DeleteDefaultService
+"""""""""""""""""""""
+
+Custom setting, this allows you to NOT define a ECS Service along with the task, therefore you will only get the TaskDefinition
+created.
+
+.. _AWS CFN Events Rules definitions: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html
+.. _Task Rule target definition: https://docs.aws.amazon.com/eventbridge/latest/APIReference/API_PutTargets.html
diff --git a/ecs_composex/common/compose_resources.py b/ecs_composex/common/compose_resources.py
@@ -145,6 +145,7 @@ def handle_families_targets_expansion(self, service, settings):
                         False,
                         [the_service],
                         service["access"],
+                        service,
                     )
                 )
 
@@ -172,6 +173,7 @@ def set_services_targets(self, settings):
                         True,
                         settings.families[service_name].services,
                         service["access"],
+                        service,
                     )
                 )
             elif service_name in settings.families and service_name in [

diff --git a/ecs_composex/common/compose_services.py b/ecs_composex/common/compose_services.py
@@ -952,6 +952,7 @@ def handle_iam(self):
         for setting in iam_settings:
             for key in valid_keys:
                 self.sort_iam_settings(key, setting)
+
         self.set_secrets_access()
 
     def handle_permission_boundary(self, prop_key):

diff --git a/ecs_composex/ecs/ecs_iam.py b/ecs_composex/ecs/ecs_iam.py
@@ -44,6 +44,7 @@ def add_service_roles(template):
         Description=Sub(
             f"Execution role for ${{{SERVICE_NAME_T}}} in ${{{CLUSTER_NAME_T}}}"
         ),
+        ManagedPolicyArns=[],
         Policies=[
             Policy(
                 PolicyName=Sub("EcsExecRole"),
@@ -99,15 +100,13 @@ def add_service_roles(template):
             )
         ],
     )
-    policies = []
-    managed_policies = []
     Role(
         TASK_ROLE_T,
         template=template,
         AssumeRolePolicyDocument=service_role_trust_policy("ecs-tasks"),
         Description=Sub(f"TaskRole - ${{{SERVICE_NAME_T}}} in ${{{CLUSTER_NAME_T}}}"),
-        ManagedPolicyArns=managed_policies,
-        Policies=policies,
+        ManagedPolicyArns=[],
+        Policies=[],
     )
 
 

diff --git a/ecs_composex/ecs/ecs_service.py b/ecs_composex/ecs/ecs_service.py
@@ -269,9 +269,10 @@ def generate_service_template_outputs(family):
                     ecs_params.SERVICE_GROUP_ID_T,
                     "GroupId",
                     GetAtt(ecs_params.SG_T, "GroupId"),
-                )
+                ),
+                (ecs_params.TASK_T, ecs_params.TASK_T, Ref(family.task_definition)),
             ],
-            duplicate_attr=True,
+            duplicate_attr=False,
             export=False,
         ).outputs
     )

diff --git a/ecs_composex/ecs_composex.py b/ecs_composex/ecs_composex.py
@@ -55,6 +55,7 @@
     CLUSTER_NAME,
     CLUSTER_T as ROOT_CLUSTER_NAME,
     CREATE_CLUSTER,
+    FARGATE_VERSION,
 )
 from ecs_composex.ecs.ecs_conditions import CREATE_CLUSTER_CON_T, CREATE_CLUSTER_CON
 from ecs_composex.vpc import vpc_params
@@ -75,6 +76,7 @@
     "s3",
     "elbv2",
     "docdb",
+    "events",
 ]
 
 SUPPORTED_X_MODULES = [f"{X_KEY}{mod_name}" for mod_name in SUPPORTED_X_MODULE_NAMES]
@@ -337,7 +339,7 @@ def init_root_template():
 
     template = build_template(
         "Root template generated via ECS ComposeX",
-        [USE_FLEET, USE_ONDEMAND, CLUSTER_NAME, CREATE_CLUSTER],
+        [USE_FLEET, USE_ONDEMAND, CLUSTER_NAME, CREATE_CLUSTER, FARGATE_VERSION],
     )
     template.add_condition(CREATE_CLUSTER_CON_T, CREATE_CLUSTER_CON)
     return template

diff --git a/ecs_composex/events/__init__.py b/ecs_composex/events/__init__.py
@@ -0,0 +1,16 @@
+#  -*- coding: utf-8 -*-
+#   ECS ComposeX <https://github.com/lambda-my-aws/ecs_composex>
+#   Copyright (C) 2020  John Mille <[email protected]>
+#  #
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#  #
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#  #
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.