Python39 support for samcli (aws#354)

* Python39 support for samcli

* Updated reproducible-linux.txt and lambda_build_container.py for test purpose

* Revert files after testing

* updated integ test

* updated appveyor

* updated to appveyor

.pylintrc

@@ -64,6 +64,9 @@ confidence=
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
 disable=
+    C0103, # Class constant name doesn't conform to UPPER_CASE naming
+    R1729, # Use a generator instead '%s(%s)' Comprehension inside of 'any' or 'all' is unnecessary. A generator would be sufficient and faster.
+    R0801, # Similar lines in %s files
     W0613, # Unused argument %r
     W0640, # Cell variable %s defined in loop A variable used in a closure is defined in a loop
     R0902, # Too many instance attributes (%s/%s)

appveyor-windows-build-python.yml

@@ -58,6 +58,7 @@ install:
     # python is python3.7
     - "python -m pip install --upgrade pip"
     - "C:\\Python38\\python.exe -m pip install --upgrade pip"
+    - "C:\\Python39\\python.exe -m pip install --upgrade pip"
 
     # Create new virtual environment with chosen python version and activate it
     - "python -m virtualenv venv"

appveyor.yml

@@ -1,7 +1,7 @@
 version: 1.0.{build}
 image: 
   - Ubuntu
-  - Visual Studio 2017
+  - Visual Studio 2019
 
 environment:
   AWS_DEFAULT_REGION: us-east-1
@@ -15,6 +15,7 @@ environment:
       NOSE_PARAMETERIZED_NO_WARN: 1
       INSTALL_PY_37_PIP: 1
       INSTALL_PY_38_PIP: 1
+      INSTALL_PY_39_PIP: 1
       AWS_S3: 'AWS_S3_36'
       AWS_ECR: 'AWS_ECR_36'
       APPVEYOR_CONSOLE_DISABLE_PTY: true
@@ -26,6 +27,7 @@ environment:
       NOSE_PARAMETERIZED_NO_WARN: 1
       INSTALL_PY_36_PIP: 1
       INSTALL_PY_38_PIP: 1
+      INSTALL_PY_39_PIP: 1
       AWS_S3: 'AWS_S3_37'
       AWS_ECR: 'AWS_ECR_37'
       APPVEYOR_CONSOLE_DISABLE_PTY: true
@@ -37,15 +39,28 @@ environment:
       NOSE_PARAMETERIZED_NO_WARN: 1
       INSTALL_PY_36_PIP: 1
       INSTALL_PY_37_PIP: 1
+      INSTALL_PY_39_PIP: 1
       AWS_S3: 'AWS_S3_38'
       AWS_ECR: 'AWS_ECR_38'
       APPVEYOR_CONSOLE_DISABLE_PTY: true
 
+    - PYTHON_HOME: "C:\\Python39-x64"
+      PYTHON_VERSION: '3.9'
+      PYTHON_ARCH: '64'
+      RUN_SMOKE: 1
+      NOSE_PARAMETERIZED_NO_WARN: 1
+      INSTALL_PY_36_PIP: 1
+      INSTALL_PY_37_PIP: 1
+      INSTALL_PY_38_PIP: 1
+      AWS_S3: 'AWS_S3_39'
+      AWS_ECR: 'AWS_ECR_39'
+      APPVEYOR_CONSOLE_DISABLE_PTY: true
+
 for:
   - 
     matrix:
       only:
-        - image: Visual Studio 2017
+        - image: Visual Studio 2019
 
     install:
       - "SET PATH=%PYTHON_HOME%;%PATH%"
@@ -113,6 +128,8 @@ for:
     install:
       # AppVeyor's apt-get cache might be outdated, and the package could potentially be 404.
       - sh: "sudo apt-get update"
+      # apt repo for python3.9 installation
+      - sh: "sudo add-apt-repository ppa:deadsnakes/ppa"
 
       - sh: "gvm use go1.13"
       - sh: "echo $PATH"
@@ -138,16 +155,19 @@ for:
       - sh: "sudo apt-get -y install python2.7"
       - sh: "sudo apt-get -y install python3.7"
       - sh: "sudo apt-get -y install python3.8"
+      - sh: "sudo apt-get -y install python3.9"
 
+      - sh: "which python3.9"
       - sh: "which python3.8"
       - sh: "which python3.6"
       - sh: "which python3.7"
       - sh: "which python2.7"
 
-      - sh: "PATH=$PATH:/usr/bin/python3.8:/usr/bin/python3.7"
+      - sh: "PATH=$PATH:/usr/bin/python3.9:/usr/bin/python3.8:/usr/bin/python3.7"
       - sh: "curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
 
       - sh: "sudo apt-get -y install python3-distutils"
+      - ps: "If ($env:INSTALL_PY_39_PIP) {python3.9 get-pip.py}"
       - ps: "If ($env:INSTALL_PY_38_PIP) {python3.8 get-pip.py --user}"
       - ps: "If ($env:INSTALL_PY_37_PIP) {python3.7 get-pip.py --user}"
       - ps: "If ($env:INSTALL_PY_36_PIP) {python3.6 get-pip.py --user}"

requirements/pre-dev.txt

@@ -1 +1 @@
-pylint~=2.6.0
+pylint~=2.7.0

samcli/commands/build/command.py

@@ -38,7 +38,7 @@
 \b
 Supported Runtimes
 ------------------
-1. Python 2.7, 3.6, 3.7, 3.8 using PIP\n
+1. Python 2.7, 3.6, 3.7, 3.8 3.9 using PIP\n
 2. Nodejs 14.x, 12.x, 10.x, 8.10, 6.10 using NPM\n
 3. Ruby 2.5 using Bundler\n
 4. Java 8, Java 11 using Gradle and Maven\n
@@ -316,7 +316,6 @@ def do_cli(  # pylint: disable=too-many-locals, too-many-statements
 
         try:
             artifacts = builder.build()
-
             stack_output_template_path_by_stack_path = {
                 stack.stack_path: stack.get_output_template_path(ctx.build_dir) for stack in ctx.stacks
             }

samcli/lib/build/workflow_config.py

@@ -151,6 +151,7 @@ def get_layer_subfolder(build_workflow: str) -> str:
         "python3.6": "python",
         "python3.7": "python",
         "python3.8": "python",
+        "python3.9": "python",
         "nodejs4.3": "nodejs",
         "nodejs6.10": "nodejs",
         "nodejs8.10": "nodejs",
@@ -210,6 +211,7 @@ def get_workflow_config(
         "python3.6": BasicWorkflowSelector(PYTHON_PIP_CONFIG),
         "python3.7": BasicWorkflowSelector(PYTHON_PIP_CONFIG),
         "python3.8": BasicWorkflowSelector(PYTHON_PIP_CONFIG),
+        "python3.9": BasicWorkflowSelector(PYTHON_PIP_CONFIG),
         "nodejs10.x": BasicWorkflowSelector(NODEJS_NPM_CONFIG),
         "nodejs12.x": BasicWorkflowSelector(NODEJS_NPM_CONFIG),
         "nodejs14.x": BasicWorkflowSelector(NODEJS_NPM_CONFIG),

samcli/lib/deploy/deployer.py

@@ -83,7 +83,8 @@ def __init__(self, cloudformation_client, changeset_prefix="samcli-deploy"):
         # Maximum number of attempts before raising exception back up the chain.
         self.max_attempts = 3
         self.deploy_color = DeployColor()
-
+
+    # pylint: disable=inconsistent-return-statements
     def has_stack(self, stack_name):
         """
         Checks if a CloudFormation stack with given name exists

samcli/local/common/runtime_template.py

@@ -15,7 +15,7 @@
 RUNTIME_DEP_TEMPLATE_MAPPING = {
     "python": [
         {
-            "runtimes": ["python3.8", "python3.7", "python3.6", "python2.7"],
+            "runtimes": ["python3.9", "python3.8", "python3.7", "python3.6", "python2.7"],
             "dependency_manager": "pip",
             "init_location": os.path.join(_templates, "cookiecutter-aws-sam-hello-python"),
             "build": True,
@@ -79,6 +79,7 @@ def get_local_lambda_images_location(mapping, runtime):
 
 
 RUNTIME_TO_DEPENDENCY_MANAGERS = {
+    "python3.9": ["pip"],
     "python3.8": ["pip"],
     "python3.7": ["pip"],
     "python3.6": ["pip"],
@@ -113,7 +114,7 @@ def get_local_lambda_images_location(mapping, runtime):
 INIT_RUNTIMES = [
     # latest of each runtime version
     "nodejs14.x",
-    "python3.8",
+    "python3.9",
     "ruby2.7",
     "go1.x",
     "java11",
@@ -122,6 +123,7 @@ def get_local_lambda_images_location(mapping, runtime):
     "nodejs12.x",
     "nodejs10.x",
     # older python runtimes
+    "python3.8",
     "python3.7",
     "python3.6",
     "python2.7",
@@ -138,6 +140,7 @@ def get_local_lambda_images_location(mapping, runtime):
     "amazon/nodejs14.x-base",
     "amazon/nodejs12.x-base",
     "amazon/nodejs10.x-base",
+    "amazon/python3.9-base",
     "amazon/python3.8-base",
     "amazon/python3.7-base",
     "amazon/python3.6-base",
@@ -162,4 +165,5 @@ def get_local_lambda_images_location(mapping, runtime):
     "python3.7": "Python36",
     "python3.6": "Python36",
     "python3.8": "Python36",
+    "python3.9": "Python36",
 }

samcli/local/docker/lambda_debug_settings.py

@@ -156,6 +156,10 @@ def get_debug_settings(debug_port, debug_args_list, _container_env_vars, runtime
                 entry + ["/var/lang/bin/python3.8"] + debug_args_list + ["/var/runtime/bootstrap.py"],
                 container_env_vars=_container_env_vars,
             ),
+            Runtime.python39.value: lambda: DebugSettings(
+                entry + ["/var/lang/bin/python3.9"] + debug_args_list + ["/var/runtime/bootstrap.py"],
+                container_env_vars=_container_env_vars,
+            ),
         }
         try:
             return entrypoint_mapping[runtime]()

samcli/local/docker/lambda_image.py

@@ -30,6 +30,7 @@ class Runtime(Enum):
     python36 = "python3.6"
     python37 = "python3.7"
     python38 = "python3.8"
+    python39 = "python3.9"
     ruby25 = "ruby2.5"
     ruby27 = "ruby2.7"
     java8 = "java8"

setup.py

@@ -71,6 +71,7 @@ def read_version():
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Topic :: Internet",
         "Topic :: Software Development :: Build Tools",
         "Topic :: Utilities",

tests/integration/buildcmd/test_build_cmd.py

@@ -54,7 +54,7 @@ class TestBuildCommand_PythonFunctions_Images(BuildIntegBase):
 
     FUNCTION_LOGICAL_ID_IMAGE = "ImageFunction"
 
-    @parameterized.expand([("3.6", False), ("3.7", False), ("3.8", False)])
+    @parameterized.expand([("3.6", False), ("3.7", False), ("3.8", False), ("3.9", False)])
     @pytest.mark.flaky(reruns=3)
     def test_with_default_requirements(self, runtime, use_container):
         overrides = {
@@ -96,13 +96,15 @@ class TestBuildCommand_PythonFunctions(BuildIntegBase):
             ("python3.6", "Python", False),
             ("python3.7", "Python", False),
             ("python3.8", "Python", False),
+            ("python3.9", "Python", False),
             # numpy 1.20.3 (in PythonPEP600/requirements.txt) only support python 3.7+
             ("python3.7", "PythonPEP600", False),
             ("python3.8", "PythonPEP600", False),
             ("python2.7", "Python", "use_container"),
             ("python3.6", "Python", "use_container"),
             ("python3.7", "Python", "use_container"),
             ("python3.8", "Python", "use_container"),
+            ("python3.9", "Python", "use_container"),
         ]
     )
     @pytest.mark.flaky(reruns=3)
@@ -1187,12 +1189,12 @@ class TestBuildWithDedupBuilds(DedupBuildIntegBase):
             ),
             (False, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"),
             (False, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"),
-            (False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.8"),
+            (False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"),
             (False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"),
             # container
             (True, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"),
             (True, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"),
-            (True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.8"),
+            (True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"),
             (True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"),
         ]
     )
@@ -1310,12 +1312,12 @@ class TestBuildWithCacheBuilds(CachedBuildIntegBase):
             ),
             (False, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"),
             (False, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"),
-            (False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.8"),
+            (False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"),
             (False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"),
             # container
             (True, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"),
             (True, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"),
-            (True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.8"),
+            (True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"),
             (True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"),
         ]
     )
@@ -1366,12 +1368,12 @@ class TestParallelBuilds(DedupBuildIntegBase):
             ),
             (False, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"),
             (False, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"),
-            (False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.8"),
+            (False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"),
             (False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"),
             # container
             (True, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"),
             (True, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"),
-            (True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.8"),
+            (True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"),
             (True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"),
         ]
     )