Merge branch 'delete-template-artifacts' into delete-cf-s3-methods

samcli/commands/delete/delete_context.py

@@ -4,6 +4,8 @@
 
 import boto3
 
+
+import docker
 import click
 from click import confirm
 from click import prompt
@@ -13,11 +15,16 @@
 from samcli.lib.package.s3_uploader import S3Uploader
 from samcli.lib.package.artifact_exporter import mktempfile, get_cf_template_name
 
+from samcli.yamlhelper import yaml_parse
+
+from samcli.lib.package.artifact_exporter import Template
+from samcli.lib.package.ecr_uploader import ECRUploader
+from samcli.lib.package.uploaders import Uploaders
+
 CONFIG_COMMAND = "deploy"
 CONFIG_SECTION = "parameters"
 TEMPLATE_STAGE = "Original"
 
-
 class DeleteContext:
     def __init__(self, stack_name: str, region: str, profile: str, config_file: str, config_env: str):
         self.stack_name = stack_name
@@ -29,6 +36,7 @@ def __init__(self, stack_name: str, region: str, profile: str, config_file: str,
         self.s3_prefix = None
         self.cf_utils = None
         self.s3_uploader = None
+        self.uploaders = None
         self.cf_template_file_name = None
         self.delete_artifacts_folder = None
         self.delete_cf_template_file = None
@@ -70,6 +78,7 @@ def delete(self):
         """
         template = self.cf_utils.get_stack_template(self.stack_name, TEMPLATE_STAGE)
         template_str = template.get("TemplateBody", None)
+        template_dict = yaml_parse(template_str)
 
         if self.s3_bucket and self.s3_prefix and template_str:
             self.delete_artifacts_folder = confirm(
@@ -92,9 +101,14 @@ def delete(self):
 
         # Delete the primary stack
         self.cf_utils.delete_stack(stack_name=self.stack_name)
-
+        self.cf_utils.wait_for_delete(self.stack_name)
+
         click.echo(f"\n\t- Deleting Cloudformation stack {self.stack_name}")
-
+
+        # Delete the artifacts
+        template = Template(None, None, self.uploaders, None)
+        template.delete(template_dict)
+
         # Delete the CF template file in S3
         if self.delete_cf_template_file:
             self.s3_uploader.delete_artifact(remote_path=self.cf_template_file_name)
@@ -124,8 +138,14 @@ def run(self):
             )
 
             s3_client = boto3.client("s3", region_name=self.region if self.region else None, config=boto_config)
+            ecr_client = boto3.client("ecr", region_name=self.region if self.region else None, config=boto_config)
 
             self.s3_uploader = S3Uploader(s3_client=s3_client, bucket_name=self.s3_bucket, prefix=self.s3_prefix)
+
+            docker_client = docker.from_env()
+            ecr_uploader = ECRUploader(docker_client, ecr_client, None, None)
+
+            self.uploaders = Uploaders(self.s3_uploader, ecr_uploader)
             self.cf_utils = CfUtils(cloudformation_client)
 
             is_deployed = self.cf_utils.has_stack(stack_name=self.stack_name)

samcli/commands/package/exceptions.py

@@ -62,6 +62,28 @@ def __init__(self, resource_id, property_name, property_value, ex):
         )
 
 
+class DeleteArtifactFailedError(UserException):
+    def __init__(self, resource_id, property_name, ex):
+        self.resource_id = resource_id
+        self.property_name = property_name
+        self.ex = ex
+
+        message_fmt = (
+            "Unable to delete artifact referenced "
+            "by {property_name} parameter of {resource_id} resource."
+            "\n"
+            "{ex}"
+        )
+
+        super().__init__(
+            message=message_fmt.format(
+                property_name=self.property_name,
+                resource_id=self.resource_id,
+                ex=self.ex,
+            )
+        )
+
+
 class ImageNotFoundError(UserException):
     def __init__(self, resource_id, property_name):
         self.resource_id = resource_id

samcli/lib/delete/cf_utils.py

@@ -4,10 +4,12 @@
 
 import logging
 
+
 from typing import Dict
-from botocore.exceptions import ClientError, BotoCoreError
+from botocore.exceptions import ClientError, BotoCoreError, WaiterError
 from samcli.commands.delete.exceptions import DeleteFailedError, FetchTemplateFailedError
 
+
 LOG = logging.getLogger(__name__)
 
 
@@ -102,3 +104,26 @@ def delete_stack(self, stack_name: str):
             # We don't know anything about this exception. Don't handle
             LOG.error("Failed to delete stack. ", exc_info=e)
             raise e
+
+    def wait_for_delete(self, stack_name):
+        """
+        Waits until the delete stack completes
+
+        :param stack_name:   Stack name
+        """
+
+        # Wait for Delete to Finish
+        waiter = self._client.get_waiter("stack_delete_complete")
+        # Poll every 5 seconds.
+        waiter_config = {"Delay": 5}
+        try:
+            waiter.wait(StackName=stack_name, WaiterConfig=waiter_config)
+        except WaiterError as ex:
+
+            resp = ex.last_response
+            status = resp["Status"]
+            reason = resp["StatusReason"]
+
+            raise DeleteFailedError(
+                stack_name=stack_name, msg="ex: {0} Status: {1}. Reason: {2}".format(ex, status, reason)
+            ) from ex

samcli/lib/package/artifact_exporter.py

@@ -130,21 +130,22 @@ def __init__(
         """
         Reads the template and makes it ready for export
         """
-        if not (is_local_folder(parent_dir) and os.path.isabs(parent_dir)):
-            raise ValueError("parent_dir parameter must be " "an absolute path to a folder {0}".format(parent_dir))
+        if template_path and parent_dir:
+            if not (is_local_folder(parent_dir) and os.path.isabs(parent_dir)):
+                raise ValueError("parent_dir parameter must be " "an absolute path to a folder {0}".format(parent_dir))
 
-        abs_template_path = make_abs_path(parent_dir, template_path)
-        template_dir = os.path.dirname(abs_template_path)
+            abs_template_path = make_abs_path(parent_dir, template_path)
+            template_dir = os.path.dirname(abs_template_path)
 
-        with open(abs_template_path, "r") as handle:
-            template_str = handle.read()
+            with open(abs_template_path, "r") as handle:
+                template_str = handle.read()
 
-        self.template_dict = yaml_parse(template_str)
-        self.template_dir = template_dir
+            self.template_dict = yaml_parse(template_str)
+            self.template_dir = template_dir
+            self.code_signer = code_signer
         self.resources_to_export = resources_to_export
         self.metadata_to_export = metadata_to_export
         self.uploaders = uploaders
-        self.code_signer = code_signer
 
     def _export_global_artifacts(self, template_dict: Dict) -> Dict:
         """
@@ -235,3 +236,27 @@ def export(self) -> Dict:
                 exporter.export(resource_id, resource_dict, self.template_dir)
 
         return self.template_dict
+
+    def delete(self, template_dict):
+        self.template_dict = template_dict
+
+        if "Resources" not in self.template_dict:
+            return self.template_dict
+
+        self._apply_global_values()
+
+        for resource_id, resource in self.template_dict["Resources"].items():
+
+            resource_type = resource.get("Type", None)
+            resource_dict = resource.get("Properties", {})
+            resource_deletion_policy = resource.get("DeletionPolicy", None)
+            if resource_deletion_policy != "Retain":
+                for exporter_class in self.resources_to_export:
+                    if exporter_class.RESOURCE_TYPE != resource_type:
+                        continue
+                    if resource_dict.get("PackageType", ZIP) != exporter_class.ARTIFACT_TYPE:
+                        continue
+                    # Delete code resources
+                    exporter = exporter_class(self.uploaders, None)
+                    exporter.delete(resource_id, resource_dict)
+        return self.template_dict

samcli/lib/package/ecr_uploader.py

@@ -5,12 +5,19 @@
 import base64
 import os
 
+import click
 import botocore
 import docker
 
 from docker.errors import BuildError, APIError
 
-from samcli.commands.package.exceptions import DockerPushFailedError, DockerLoginFailedError, ECRAuthorizationError
+from samcli.commands.package.exceptions import (
+    DockerPushFailedError,
+    DockerLoginFailedError,
+    ECRAuthorizationError,
+    ImageNotFoundError,
+    DeleteArtifactFailedError,
+)
 from samcli.lib.package.image_utils import tag_translation
 from samcli.lib.package.stream_cursor_utils import cursor_up, cursor_left, cursor_down, clear_line
 from samcli.lib.utils.osutils import stderr
@@ -83,6 +90,30 @@ def upload(self, image, resource_name):
 
         return f"{repository}:{_tag}"
 
+    def delete_artifact(self, image_uri: str, resource_id: str, property_name: str):
+        try:
+            repo_image_tag = image_uri.split("/")[1].split(":")
+            repository = repo_image_tag[0]
+            image_tag = repo_image_tag[1]
+            resp = self.ecr_client.batch_delete_image(
+                repositoryName=repository,
+                imageIds=[
+                    {"imageTag": image_tag},
+                ],
+            )
+            if resp["failures"]:
+                # Image not found
+                image_details = resp["failures"][0]
+                if image_details["failureCode"] == "ImageNotFound":
+                    LOG.debug("ImageNotFound Exception : ")
+                    raise ImageNotFoundError(resource_id, property_name)
+
+            click.echo(f"- Deleting ECR image {image_tag} in repository {repository}")
+
+        except botocore.exceptions.ClientError as ex:
+            # Handle Client errors such as RepositoryNotFoundException or InvalidParameterException
+            raise DeleteArtifactFailedError(resource_id=resource_id, property_name=property_name, ex=ex) from ex
+
     # TODO: move this to a generic class to allow for streaming logs back from docker.
     def _stream_progress(self, logs):
         """

samcli/lib/package/packageable_resources.py

@@ -22,6 +22,7 @@
     upload_local_image_artifacts,
     is_s3_protocol_url,
     is_path_value_valid,
+    is_ecr_url,
 )
 
 from samcli.commands._utils.resources import (
@@ -79,6 +80,9 @@ def export(self, resource_id, resource_dict, parent_dir):
     def do_export(self, resource_id, resource_dict, parent_dir):
         pass
 
+    def delete(self, resource_id, resource_dict):
+        pass
+
 
 class ResourceZip(Resource):
     """
@@ -154,6 +158,18 @@ def do_export(self, resource_id, resource_dict, parent_dir):
             )
         set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url)
 
+    def delete(self, resource_id, resource_dict):
+        """
+        Delete the S3 artifact using S3 url referenced by PROPERTY_NAME
+        """
+        if resource_dict is None:
+            return
+        resource_path = resource_dict[self.PROPERTY_NAME]
+        parsed_s3_url = self.uploader.parse_s3_url(resource_path)
+        if not self.uploader.bucket_name:
+            self.uploader.bucket_name = parsed_s3_url["Bucket"]
+        self.uploader.delete_artifact(parsed_s3_url["Key"], True)
+
 
 class ResourceImageDict(Resource):
     """
@@ -197,6 +213,19 @@ def do_export(self, resource_id, resource_dict, parent_dir):
         )
         set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, {self.EXPORT_PROPERTY_CODE_KEY: uploaded_url})
 
+    def delete(self, resource_id, resource_dict):
+        """
+        Delete the ECR artifact using ECR url in PROPERTY_NAME referenced by EXPORT_PROPERTY_CODE_KEY
+        """
+        if resource_dict is None:
+            return
+
+        remote_path = resource_dict[self.PROPERTY_NAME][self.EXPORT_PROPERTY_CODE_KEY]
+        if is_ecr_url(remote_path):
+            self.uploader.delete_artifact(
+                image_uri=remote_path, resource_id=resource_id, property_name=self.PROPERTY_NAME
+            )
+
 
 class ResourceImage(Resource):
     """
@@ -238,6 +267,19 @@ def do_export(self, resource_id, resource_dict, parent_dir):
         )
         set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url)
 
+    def delete(self, resource_id, resource_dict):
+        """
+        Delete the ECR artifact using ECR url referenced by property_name
+        """
+        if resource_dict is None:
+            return
+
+        remote_path = resource_dict[self.PROPERTY_NAME]
+        if is_ecr_url(remote_path):
+            self.uploader.delete_artifact(
+                image_uri=remote_path, resource_id=resource_id, property_name=self.PROPERTY_NAME
+            )
+
 
 class ResourceWithS3UrlDict(ResourceZip):
     """
@@ -269,6 +311,21 @@ def do_export(self, resource_id, resource_dict, parent_dir):
         )
         set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, parsed_url)
 
+    def delete(self, resource_id, resource_dict):
+        """
+        Delete the S3 artifact using S3 url in the dict PROPERTY_NAME
+        using the bucket at BUCKET_NAME_PROPERTY and key at OBJECT_KEY_PROPERTY
+        """
+        if resource_dict is None:
+            return
+        resource_path = resource_dict[self.PROPERTY_NAME]
+        s3_bucket = resource_path[self.BUCKET_NAME_PROPERTY]
+        key = resource_path[self.OBJECT_KEY_PROPERTY]
+
+        if not self.uploader.bucket_name:
+            self.uploader.bucket_name = s3_bucket
+        self.uploader.delete_artifact(remote_path=key, is_key=True)
+
 
 class ServerlessFunctionResource(ResourceZip):
     RESOURCE_TYPE = AWS_SERVERLESS_FUNCTION

samcli/lib/package/s3_uploader.py

@@ -86,7 +86,7 @@ def upload(self, file_name: str, remote_path: str) -> str:
 
         # Check if a file with same data exists
         if not self.force_upload and self.file_exists(remote_path):
-            LOG.debug("File with same data is already exists at %s. " "Skipping upload", remote_path)
+            LOG.info("File with same data already exists at %s, skipping upload", remote_path)
             return self.make_url(remote_path)
 
         try: