feat(images): SAM CLI support for Lambda Images (#2425)

* feat(images): SAM CLI support for Lambda Images

* fix(tests): remove teardown on layer tests

* fix(coverage): set coverage to 94%

.pylintrc

@@ -59,7 +59,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=R0201,W0613,W0640,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904,R0914,C0301,C0415
+disable=R0201,W0613,W0640,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904,R0914,R0915,C0301,C0415
 
 
 [REPORTS]

MANIFEST.in

@@ -1,9 +1,10 @@
 include LICENSE
+include THIRD-PARTY-LICENSES
 include requirements/base.txt
 include requirements/dev.txt
 include samcli/local/go-bootstrap/aws-lambda-go
-include samcli/local/rapid/init
-include samcli/local/preview-rapid/init
+include samcli/local/rapid/aws-lambda-rie
+recursive-include samcli/lib/init/image_templates *
 recursive-include samcli/lib/init/templates *
 recursive-include samcli/lib *.json
 recursive-include samcli/lib/generated_sample_events *.json

NOTICE

@@ -1,6 +1,6 @@
 AWS SAM CLI
 Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
-Contents of the samcli/local/rapid and samcli/local/go-bootstrap folder forked from the original Open Source MIT licensed project lambci/docker-lambda: https://github.com/lambci/docker-lambda/
+Contents of the samcli/local/go-bootstrap folder forked from the original Open Source MIT licensed project lambci/docker-lambda: https://github.com/lambci/docker-lambda/
 Copyright 2016 Michael Hart and LambCI contributors
 Portions Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

THIRD-PARTY-LICENSES

@@ -180,10 +180,7 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-------
-
-Contents of the samcli/local/rapid and samcli/local/go-bootstrap folder forked from the original Open Source MIT licensed project lambci/docker-lambda: https://github.com/lambci/docker-lambda/
-
+Contents of the samcli/local/go-bootstrap folder forked from the original Open Source MIT licensed project lambci/docker-lambda: https://github.com/lambci/docker-lambda/
 Copyright 2016 Michael Hart and LambCI contributors
 Portions Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 

appveyor-windows.yml

@@ -26,6 +26,7 @@ environment:
   HOMEPATH: 'C:\Users\appveyor'
   NOSE_PARAMETERIZED_NO_WARN: 1
   AWS_S3: 'AWS_S3_37_WIN'
+  AWS_ECR: 'AWS_ECR_37'
   APPVEYOR_CONSOLE_DISABLE_PTY: true
 
 init:

appveyor.yml

@@ -16,6 +16,7 @@ environment:
       INSTALL_PY_37_PIP: 1
       INSTALL_PY_38_PIP: 1
       AWS_S3: 'AWS_S3_36'
+      AWS_ECR: 'AWS_ECR_36'
       APPVEYOR_CONSOLE_DISABLE_PTY: true
 
     - PYTHON_HOME: "C:\\Python37-x64"
@@ -26,6 +27,7 @@ environment:
       INSTALL_PY_36_PIP: 1
       INSTALL_PY_38_PIP: 1
       AWS_S3: 'AWS_S3_37'
+      AWS_ECR: 'AWS_ECR_37'
       APPVEYOR_CONSOLE_DISABLE_PTY: true
 
     - PYTHON_HOME: "C:\\Python38-x64"
@@ -36,6 +38,7 @@ environment:
       INSTALL_PY_36_PIP: 1
       INSTALL_PY_37_PIP: 1
       AWS_S3: 'AWS_S3_38'
+      AWS_ECR: 'AWS_ECR_38'
       APPVEYOR_CONSOLE_DISABLE_PTY: true
 
 for:
@@ -74,7 +77,7 @@ for:
     test_script:
       # Activate virtualenv again on windows
       - "venv\\Scripts\\activate"
-      - "pytest --cov samcli --cov-report term-missing --cov-fail-under 95 tests/unit"
+      - "pytest --cov samcli --cov-report term-missing --cov-fail-under 94 tests/unit"
       - "pylint --rcfile .pylintrc samcli"
       - "pytest -n 4 tests/functional"
 
@@ -127,7 +130,7 @@ for:
       - "pip install -e \".[dev]\""
 
     test_script:
-      - "pytest --cov samcli --cov-report term-missing --cov-fail-under 95 tests/unit"
+      - "pytest --cov samcli --cov-report term-missing --cov-fail-under 94 tests/unit"
       - "pylint --rcfile .pylintrc samcli"
       - "pytest -n 4 tests/functional"
 

requirements/base.txt

@@ -6,7 +6,7 @@ boto3~=1.14.23
 jmespath~=0.10.0
 PyYAML~=5.3
 cookiecutter~=1.7.2
-aws-sam-translator==1.31.0
+aws-sam-translator==1.32.0
 #docker minor version updates can include breaking changes. Auto update micro version only.
 docker~=4.2.0
 dateparser~=0.7

samcli/__init__.py

@@ -2,4 +2,4 @@
 SAM CLI version
 """
 
-__version__ = "1.12.0"
+__version__ = "1.13.0"

samcli/commands/_utils/options.py

@@ -12,6 +12,7 @@
 from samcli.commands._utils.template import get_template_data, TemplateNotFoundException
 from samcli.cli.types import CfnParameterOverridesType, CfnMetadataType, CfnTags, SigningProfilesOptionType
 from samcli.commands._utils.custom_options.option_nargs import OptionNargs
+from samcli.commands._utils.template import get_template_artifacts_format
 
 _TEMPLATE_OPTION_DEFAULT_VALUE = "template.[yaml|yml]"
 DEFAULT_STACK_NAME = "sam-app"
@@ -93,6 +94,76 @@ def guided_deploy_stack_name(ctx, param, provided_value):
     return provided_value if provided_value else DEFAULT_STACK_NAME
 
 
+def artifact_callback(ctx, param, provided_value, artifact):
+    """
+    Provide an error if there are zip/image artifact based resources, and an destination export destination is not specified.
+    :param ctx: Click Context
+    :param param: Param name
+    :param provided_value: Value provided by Click, it would be the value provided by the user.
+    :param artifact: artifact format that is to be compared against, eg: zip, image.
+    :return: Actual value to be used in the CLI
+    """
+
+    template_file = (
+        ctx.params.get("t", False) or ctx.params.get("template_file", False) or ctx.params.get("template", False)
+    )
+
+    required = any(
+        [
+            _template_artifact == artifact
+            for _template_artifact in get_template_artifacts_format(template_file=template_file)
+        ]
+    )
+    # NOTE(sriram-mv): Explicit check for param name being s3_bucket
+    # If that is the case, check for another option called resolve_s3 to be defined.
+    # resolve_s3 option resolves for the s3 bucket automatically.
+    # NOTE(sriram-mv): Both params and default_map need to be checked, as the option can be either be
+    # passed in directly or through configuration file.
+    # If passed in through configuration file, default_map is loaded with those values.
+    if param.name == "s3_bucket" and (ctx.params.get("resolve_s3", False) or ctx.default_map.get("resolve_s3", False)):
+        pass
+    elif required and not provided_value:
+        raise click.BadOptionUsage(option_name=param.name, ctx=ctx, message=f"Missing option '{param.opts[0]}'")
+
+    return provided_value
+
+
+def resolve_s3_callback(ctx, param, provided_value, artifact, exc_set, exc_not_set):
+    """
+    S3 Bucket is only required if there are artifacts that are all zip based.
+    :param ctx: Click Context
+    :param param: Param name
+    :param provided_value: Value provided by Click, it would be the value provided by the user.
+    :param artifact: artifact format that is to be compared against, eg: zip, image.
+    :param exc_set: Exception to be thrown if both `--resolve-s3` and `--s3-bucket` are set.
+    :param exc_not_set: Exception to be thrown if either `--resolve-s3` and `--s3-bucket` are not set
+    and are required because the template contains zip based artifacts.
+    :return: Actual value to be used in the CLI
+    """
+
+    template_file = (
+        ctx.params.get("t", False) or ctx.params.get("template_file", False) or ctx.params.get("template", False)
+    )
+
+    required = any(
+        [
+            _template_artifact == artifact
+            for _template_artifact in get_template_artifacts_format(template_file=template_file)
+        ]
+    )
+    # NOTE(sriram-mv): Explicit check for s3_bucket being explicitly passed in along with `--resolve-s3`.
+    # NOTE(sriram-mv): Both params and default_map need to be checked, as the option can be either be
+    # passed in directly or through configuration file.
+    # If passed in through configuration file, default_map is loaded with those values.
+    s3_bucket_provided = ctx.params.get("s3_bucket", False) or ctx.default_map.get("s3_bucket", False)
+    if provided_value and s3_bucket_provided:
+        raise exc_set()
+    if required and not provided_value and not s3_bucket_provided:
+        raise exc_not_set()
+
+    return provided_value
+
+
 def template_common_option(f):
     """
     Common ClI option for template

samcli/commands/_utils/resources.py

@@ -2,6 +2,8 @@
 Enums for Resources and thier Location Properties, along with utlity functions
 """
 
+from collections import defaultdict
+
 AWS_SERVERLESSREPO_APPLICATION = "AWS::ServerlessRepo::Application"
 AWS_SERVERLESS_FUNCTION = "AWS::Serverless::Function"
 AWS_SERVERLESS_API = "AWS::Serverless::Api"
@@ -41,12 +43,26 @@
     AWS_STEPFUNCTIONS_STATEMACHINE: ["DefinitionS3Location"],
 }
 
+RESOURCES_WITH_IMAGE_COMPONENT = {
+    AWS_SERVERLESS_FUNCTION: ["ImageUri"],
+    AWS_LAMBDA_FUNCTION: ["Code"],
+}
+
 
 def resources_generator():
     """
     Generator to yield set of resources and their locations that are supported for package operations
     :return:
     """
-    for resource, locations in dict({**METADATA_WITH_LOCAL_PATHS, **RESOURCES_WITH_LOCAL_PATHS}).items():
-        for location in locations:
-            yield resource, location
+    _resource_property_dict = defaultdict(list)
+    for _dict in (METADATA_WITH_LOCAL_PATHS, RESOURCES_WITH_LOCAL_PATHS, RESOURCES_WITH_IMAGE_COMPONENT):
+        for key, value in _dict.items():
+            # Only add values to the list if they are different, same property name could be used with the resource
+            # to package to different locations.
+            if value not in _resource_property_dict.get(key, []):
+                _resource_property_dict[key].append(value)
+
+    for resource, location_list in _resource_property_dict.items():
+        for locations in location_list:
+            for location in locations:
+                yield resource, location

samcli/commands/_utils/template.py

@@ -10,8 +10,13 @@
 from botocore.utils import set_value_from_jmespath
 
 from samcli.commands.exceptions import UserException
+from samcli.lib.utils.packagetype import ZIP
 from samcli.yamlhelper import yaml_parse, yaml_dump
-from samcli.commands._utils.resources import METADATA_WITH_LOCAL_PATHS, RESOURCES_WITH_LOCAL_PATHS
+from samcli.commands._utils.resources import (
+    METADATA_WITH_LOCAL_PATHS,
+    RESOURCES_WITH_LOCAL_PATHS,
+    RESOURCES_WITH_IMAGE_COMPONENT,
+)
 
 
 class TemplateNotFoundException(UserException):
@@ -240,3 +245,19 @@ def get_template_parameters(template_file):
     """
     template_dict = get_template_data(template_file=template_file)
     return template_dict.get("Parameters", dict())
+
+
+def get_template_artifacts_format(template_file):
+    """
+    Get a list of template artifact formats based on PackageType
+    :param template_file:
+    :return: list of artifact formats
+    """
+
+    template_dict = get_template_data(template_file=template_file)
+    return list(
+        {
+            resource_id: resource.get("Properties", {}).get("PackageType", ZIP)
+            for resource_id, resource in template_dict.get("Resources", {}).items()
+        }.values()
+    )

samcli/commands/build/command.py

@@ -12,7 +12,7 @@
     parameter_override_option,
 )
 from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options
-from samcli.lib.build.app_builder import BuildInsideContainerError
+from samcli.lib.build.exceptions import BuildInsideContainerError
 from samcli.lib.telemetry.metrics import track_command
 from samcli.cli.cli_config_file import configuration_option, TomlProvider
 

samcli/commands/deploy/auth_utils.py

@@ -3,8 +3,8 @@
 """
 import logging
 
+from samcli.commands.deploy.transform_utils import transform_template
 from samcli.commands.local.lib.swagger.reader import SwaggerReader
-from samcli.lib.providers.sam_function_provider import SamFunctionProvider
 
 LOG = logging.getLogger(__name__)
 
@@ -30,9 +30,7 @@ def auth_per_resource(parameter_overrides, template_dict):
 
     _auth_per_resource = []
 
-    sam_functions = SamFunctionProvider(
-        template_dict=template_dict, parameter_overrides=parameter_overrides, ignore_code_extraction_warnings=True
-    )
+    sam_functions = transform_template(parameter_overrides=parameter_overrides, template_dict=template_dict)
     for sam_function in sam_functions.get_all():
         # Only check for auth if there are function events defined.
         if sam_function.events:

samcli/commands/deploy/command.py

@@ -2,6 +2,7 @@
 CLI command for "deploy" command
 """
 import logging
+from functools import partial
 
 import click
 
@@ -16,6 +17,7 @@
     no_progressbar_option,
     tags_override_option,
     template_click_option,
+    artifact_callback,
     signing_profiles_option,
 )
 from samcli.commands.deploy.utils import sanitize_parameter_overrides
@@ -69,6 +71,11 @@
     "CloudFormation template. This is required the deployments of "
     "templates sized greater than 51,200 bytes",
 )
+@click.option(
+    "--image-repository",
+    required=False,
+    help="ECR repo uri where this command uploads the image artifacts that are referenced in your template.",
+)
 @click.option(
     "--force-upload",
     required=False,
@@ -153,6 +160,7 @@ def cli(
     template_file,
     stack_name,
     s3_bucket,
+    image_repository,
     force_upload,
     no_progressbar,
     s3_prefix,
@@ -179,6 +187,7 @@ def cli(
         template_file,
         stack_name,
         s3_bucket,
+        image_repository,
         force_upload,
         no_progressbar,
         s3_prefix,
@@ -207,6 +216,7 @@ def do_cli(
     template_file,
     stack_name,
     s3_bucket,
+    image_repository,
     force_upload,
     no_progressbar,
     s3_prefix,
@@ -240,6 +250,7 @@ def do_cli(
             template_file=template_file,
             stack_name=stack_name,
             s3_bucket=s3_bucket,
+            image_repository=image_repository,
             s3_prefix=s3_prefix,
             region=region,
             profile=profile,
@@ -266,6 +277,7 @@ def do_cli(
             template_file=template_file,
             s3_bucket=guided_context.guided_s3_bucket if guided else s3_bucket,
             s3_prefix=guided_context.guided_s3_prefix if guided else s3_prefix,
+            image_repository=guided_context.guided_image_repository if guided else image_repository,
             output_template_file=output_template_file.name,
             kms_key_id=kms_key_id,
             use_json=use_json,
@@ -283,6 +295,7 @@ def do_cli(
             template_file=output_template_file.name,
             stack_name=guided_context.guided_stack_name if guided else stack_name,
             s3_bucket=guided_context.guided_s3_bucket if guided else s3_bucket,
+            image_repository=guided_context.guided_image_repository if guided else image_repository,
             force_upload=force_upload,
             no_progressbar=no_progressbar,
             s3_prefix=guided_context.guided_s3_prefix if guided else s3_prefix,

samcli/commands/deploy/deploy_context.py

@@ -47,6 +47,7 @@ def __init__(
         template_file,
         stack_name,
         s3_bucket,
+        image_repository,
         force_upload,
         no_progressbar,
         s3_prefix,
@@ -66,6 +67,7 @@ def __init__(
         self.template_file = template_file
         self.stack_name = stack_name
         self.s3_bucket = s3_bucket
+        self.image_repository = image_repository
         self.force_upload = force_upload
         self.no_progressbar = no_progressbar
         self.s3_prefix = s3_prefix
@@ -127,6 +129,7 @@ def run(self):
         print_deploy_args(
             self.stack_name,
             self.s3_bucket,
+            self.image_repository,
             region,
             self.capabilities,
             self.parameter_overrides,