Allow usage of SSL common name with deprecation.

In the comming months, the method by which every AWS SDK resolves
their endpoints, including aws-cli v1 and boto3, will be udpated with
completely different behavior. At this time,the use of the
`sslCommonName` paradigm will be fully deprecated. Additionally, it
was mainly used to support python 2.6, which has not been supported by
boto3 or the CLI since January 1, 2020. The boto team wanted to ease the
transition by raising a warning to users and providing an optional
environment variable to fully disable the behavior immediately. However,
this causes breaking changes in v1 of aws-cli so additional changes were
required in it to release the deprecation.

This commit attempts to formally maintain the `sslCommonName` paradigm
in aws-cli v1. This is accomplished by providing hardcoded, manual
overrides for each service and region combination where this parameter
is used over the standard `hostname`. The list of these service/region
combinations was compiled by creating a client for every single
combination and emitting the `endpoint_url` to a file twice. The first
using a version of botocore that uses `sslCommonName` and one that
doesn't, then diffing the two files to see which URLs changed.

While behavior for the end user will largely be the same, if they do not
disable the common name behavior by setting the environment variable
`BOTO_DISABLE_COMMONNAME` to `true`, a warning will be omitted in the
console once per session. Additional details can be found in
boto/botocore#2705.

The override occurs when the `before-building-argument-table-parser` is
omitted by an event handler. At this point, if the user has not set
`endpoint_url` in `parsed_globals` (`--endpoint-url` flag from the command
line) the endpoint that the CLI uses will be overridden to continue to
use the SSL common name. There were a few different options of which
event to run the override in, but ultimately this one was chosen because
it was the least invasive to the existing code base. As this is very low
level behavior for the CLI, minimizing the amount of changes made was
prioritized over one that might seemt to make more logical sense like
`top-level-args-parsed`, which is when the arguments passed to the
command line are actually parsed through. However, the usage of a
session instance was required  in order to get config variables like
region and partition. There is a strong possiblity that this
customization override will run twice per call, but that is a design
flaw the maintainers are aware of and have accpeted.

Additionally for the `health` service, the region parameter needed to be
overwritten because of it's use of pseudo regions like `aws-global`. In
the other cases, the region within the URL essentially always matched
the region, but since health uses the SSL common name in global pseudo
regions, this change was required as well or an exception would be
raised.

.changes/next-release/enhancement-Endpoints-23278.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "Endpoints",
+  "description": "Enforce SSL common name as default endpoint url"
+}

awscli/clidriver.py

@@ -484,7 +484,7 @@ def __call__(self, args, parsed_globals):
         event = 'before-building-argument-table-parser.%s.%s' % \
             (self._parent_name, self._name)
         self._emit(event, argument_table=self.arg_table, args=args,
-                   session=self._session)
+                   session=self._session, parsed_globals=parsed_globals)
         operation_parser = self._create_operation_parser(self.arg_table)
         self._add_help(operation_parser)
         parsed_args, remaining = operation_parser.parse_known_args(args)

awscli/customizations/commands.py

@@ -134,7 +134,7 @@ def __call__(self, args, parsed_globals):
         event = 'before-building-argument-table-parser.%s' % \
             ".".join(self.lineage_names)
         self._session.emit(event, argument_table=self._arg_table, args=args,
-                           session=self._session)
+                           session=self._session, parsed_globals=parsed_globals)
         parser = ArgTableArgParser(self.arg_table, self.subcommand_table)
         parsed_args, remaining = parser.parse_known_args(args)
 

awscli/customizations/overridesslcommonname.py

@@ -0,0 +1,138 @@
+# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+
+SSL_COMMON_NAMES = {
+    "sqs": {
+        "af-south-1": "af-south-1.queue.amazonaws.com",
+        "ap-east-1": "ap-east-1.queue.amazonaws.com",
+        "ap-northeast-1": "ap-northeast-1.queue.amazonaws.com",
+        "ap-northeast-2": "ap-northeast-2.queue.amazonaws.com",
+        "ap-northeast-3": "ap-northeast-3.queue.amazonaws.com",
+        "ap-south-1": "ap-south-1.queue.amazonaws.com",
+        "ap-southeast-1": "ap-southeast-1.queue.amazonaws.com",
+        "ap-southeast-2": "ap-southeast-2.queue.amazonaws.com",
+        "ap-southeast-3": "ap-southeast-3.queue.amazonaws.com",
+        "ca-central-1": "ca-central-1.queue.amazonaws.com",
+        "eu-central-1": "eu-central-1.queue.amazonaws.com",
+        "eu-north-1": "eu-north-1.queue.amazonaws.com",
+        "eu-south-1": "eu-south-1.queue.amazonaws.com",
+        "eu-west-1": "eu-west-1.queue.amazonaws.com",
+        "eu-west-2": "eu-west-2.queue.amazonaws.com",
+        "eu-west-3": "eu-west-3.queue.amazonaws.com",
+        "me-south-1": "me-south-1.queue.amazonaws.com",
+        "sa-east-1": "sa-east-1.queue.amazonaws.com",
+        "us-east-1": "queue.amazonaws.com",
+        "us-east-2": "us-east-2.queue.amazonaws.com",
+        "us-west-1": "us-west-1.queue.amazonaws.com",
+        "us-west-2": "us-west-2.queue.amazonaws.com",
+        "cn-north-1": "cn-north-1.queue.amazonaws.com.cn",
+        "cn-northwest-1": "cn-northwest-1.queue.amazonaws.com.cn",
+        "us-gov-west-1": "us-gov-west-1.queue.amazonaws.com",
+        "us-isob-east-1": "us-isob-east-1.queue.sc2s.sgov.gov",
+    },
+    "emr": {
+        "af-south-1": "af-south-1.elasticmapreduce.amazonaws.com",
+        "ap-east-1": "ap-east-1.elasticmapreduce.amazonaws.com",
+        "ap-northeast-1": "ap-northeast-1.elasticmapreduce.amazonaws.com",
+        "ap-northeast-2": "ap-northeast-2.elasticmapreduce.amazonaws.com",
+        "ap-northeast-3": "ap-northeast-3.elasticmapreduce.amazonaws.com",
+        "ap-south-1": "ap-south-1.elasticmapreduce.amazonaws.com",
+        "ap-southeast-1": "ap-southeast-1.elasticmapreduce.amazonaws.com",
+        "ap-southeast-2": "ap-southeast-2.elasticmapreduce.amazonaws.com",
+        "ap-southeast-3": "ap-southeast-3.elasticmapreduce.amazonaws.com",
+        "ca-central-1": "ca-central-1.elasticmapreduce.amazonaws.com",
+        "eu-north-1": "eu-north-1.elasticmapreduce.amazonaws.com",
+        "eu-south-1": "eu-south-1.elasticmapreduce.amazonaws.com",
+        "eu-west-1": "eu-west-1.elasticmapreduce.amazonaws.com",
+        "eu-west-2": "eu-west-2.elasticmapreduce.amazonaws.com",
+        "eu-west-3": "eu-west-3.elasticmapreduce.amazonaws.com",
+        "me-south-1": "me-south-1.elasticmapreduce.amazonaws.com",
+        "sa-east-1": "sa-east-1.elasticmapreduce.amazonaws.com",
+        "us-east-2": "us-east-2.elasticmapreduce.amazonaws.com",
+        "us-west-1": "us-west-1.elasticmapreduce.amazonaws.com",
+        "us-west-2": "us-west-2.elasticmapreduce.amazonaws.com",
+    },
+    "rds": {
+        "us-east-1": "rds.amazonaws.com",
+    },
+    "docdb": {
+        "us-east-1": "rds.amazonaws.com",
+    },
+    "neptune": {
+        "us-east-1": "rds.amazonaws.com",
+    },
+    "health": {
+        "aws-global": "health.us-east-1.amazonaws.com",
+        "af-south-1": "health.us-east-1.amazonaws.com",
+        "ap-east-1": "health.us-east-1.amazonaws.com",
+        "ap-northeast-1": "health.us-east-1.amazonaws.com",
+        "ap-northeast-2": "health.us-east-1.amazonaws.com",
+        "ap-northeast-3": "health.us-east-1.amazonaws.com",
+        "ap-south-1": "health.us-east-1.amazonaws.com",
+        "ap-southeast-1": "health.us-east-1.amazonaws.com",
+        "ap-southeast-2": "health.us-east-1.amazonaws.com",
+        "ap-southeast-3": "health.us-east-1.amazonaws.com",
+        "ca-central-1": "health.us-east-1.amazonaws.com",
+        "eu-central-1": "health.us-east-1.amazonaws.com",
+        "eu-north-1": "health.us-east-1.amazonaws.com",
+        "eu-south-1": "health.us-east-1.amazonaws.com",
+        "eu-west-1": "health.us-east-1.amazonaws.com",
+        "eu-west-2": "health.us-east-1.amazonaws.com",
+        "eu-west-3": "health.us-east-1.amazonaws.com",
+        "me-south-1": "health.us-east-1.amazonaws.com",
+        "sa-east-1": "health.us-east-1.amazonaws.com",
+        "us-east-1": "health.us-east-1.amazonaws.com",
+        "us-east-2": "health.us-east-1.amazonaws.com",
+        "us-west-1": "health.us-east-1.amazonaws.com",
+        "us-west-2": "health.us-east-1.amazonaws.com",
+        "cn-north-1": "health.cn-northwest-1.amazonaws.com.cn",
+        "cn-northwest-1": "health.cn-northwest-1.amazonaws.com.cn",
+        "aws-cn-global": "health.cn-northwest-1.amazonaws.com.cn",
+    },
+}
+
+REGION_TO_PARTITION_OVERRIDE = {
+    "aws-global": "aws",
+    "aws-cn-global": "aws-cn",
+}
+
+
+def register_override_ssl_common_name(cli):
+    cli.register_last(
+        "before-building-argument-table-parser", update_endpoint_url
+    )
+
+
+def update_endpoint_url(session, parsed_globals, **kwargs):
+    service = parsed_globals.command
+    endpoints = SSL_COMMON_NAMES.get(service)
+    region = session.get_config_variable("region")
+    # only change url if user has not overridden already themselves
+    if endpoints is not None and parsed_globals.endpoint_url is None:
+        endpoint_url = endpoints.get(region)
+        if endpoint_url is not None:
+            parsed_globals.endpoint_url = f"https://{endpoint_url}"
+            if service == "health":
+                _override_health_region(region, session, parsed_globals)
+
+
+def _override_health_region(region, session, parsed_globals):
+    if region in REGION_TO_PARTITION_OVERRIDE:
+        partition = REGION_TO_PARTITION_OVERRIDE[region]
+    else:
+        partition = session.get_partition_for_region(region)
+    if partition == "aws-cn":
+        parsed_globals.region = "cn-northwest-1"
+    else:
+        parsed_globals.region = "us-east-1"

awscli/customizations/rds.py

@@ -96,8 +96,10 @@ class GenerateDBAuthTokenCommand(BasicCommand):
 
     def _run_main(self, parsed_args, parsed_globals):
         rds = self._session.create_client(
-            'rds', parsed_globals.region, parsed_globals.endpoint_url,
-            parsed_globals.verify_ssl
+            'rds',
+            region_name=parsed_globals.region,
+            endpoint_url=parsed_globals.endpoint_url,
+            verify=parsed_globals.verify_ssl
         )
         token = rds.generate_db_auth_token(
             DBHostname=parsed_args.hostname,

awscli/handlers.py

@@ -91,6 +91,7 @@
 from awscli.customizations.sessionmanager import register_ssm_session
 from awscli.customizations.sms_voice import register_sms_voice_hide
 from awscli.customizations.dynamodb import register_dynamodb_paginator_fix
+from awscli.customizations.overridesslcommonname import register_override_ssl_common_name
 
 
 def awscli_initialize(event_handlers):
@@ -183,3 +184,4 @@ def awscli_initialize(event_handlers):
     register_ssm_session(event_handlers)
     register_sms_voice_hide(event_handlers)
     register_dynamodb_paginator_fix(event_handlers)
+    register_override_ssl_common_name(event_handlers)

tests/functional/test_override_ssl_common_name.py

@@ -0,0 +1,64 @@
+# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+from awscli.testutils import BaseAWSCommandParamsTest
+from awscli.customizations.overridesslcommonname import SSL_COMMON_NAMES
+
+
+class OverrideSSLCommonNameTestCase(BaseAWSCommandParamsTest):
+    def _common_name_test_cases(self):
+
+        service_ops = {
+            "sqs": "list-queues",
+            "emr": "list-clusters",
+            "rds": "describe-db-clusters",
+            "neptune": "describe-db-clusters",
+            "docdb": "describe-db-clusters",
+        }
+        for service, operation in service_ops.items():
+            for region in SSL_COMMON_NAMES[service]:
+                yield (service, operation, region)
+
+    def _assert_region_endpoint_used(
+        self, expected_region, expected_endpoint_url
+    ):
+        self.assertEqual(
+            expected_region, self.last_request_dict["context"]["client_region"]
+        )
+        self.assertEqual(
+            expected_endpoint_url,
+            self.last_request_dict["url"],
+        )
+
+    def test_set_endpoint_url_arg(self):
+        for service, operation, region in self._common_name_test_cases():
+            self.run_cmd(f"{service} {operation} --region {region}".split())
+            expected_endpoint_url = (
+                f"https://{SSL_COMMON_NAMES[service][region]}/"
+            )
+            self._assert_region_endpoint_used(region, expected_endpoint_url)
+
+    def test_override_health_endpoint_and_region(self):
+        expected_override_region = "us-east-1"
+        health_regions = SSL_COMMON_NAMES["health"]
+        for region in health_regions:
+            if region in ["cn-north-1", "cn-northwest-1", "aws-cn-global"]:
+                expected_override_region = "cn-northwest-1"
+            expected_endpoint_url = (
+                f"https://{SSL_COMMON_NAMES['health'][region]}/"
+            )
+            self.run_cmd(f"health describe-events --region {region}".split())
+            self._assert_region_endpoint_used(
+                expected_override_region,
+                expected_endpoint_url,
+            )

tests/unit/customizations/test_overridesslcommonname.py

@@ -0,0 +1,75 @@
+# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+from awscli.testutils import create_clidriver
+from awscli.customizations.overridesslcommonname import (
+    update_endpoint_url,
+    SSL_COMMON_NAMES,
+)
+
+import pytest
+import argparse
+
+
+def parameters():
+    for service, regions in SSL_COMMON_NAMES.items():
+        for region in regions:
+            yield (service, region)
+
+
+@pytest.fixture
+def parsed_globals():
+    pg = argparse.Namespace()
+    pg.endpoint_url = None
+    pg.region = None
+    return pg
+
+
+@pytest.fixture
+def session():
+    driver = create_clidriver()
+    return driver.session
+
+
+@pytest.mark.parametrize("service,region", parameters())
+def test_update_endpoint_url(parsed_globals, session, service, region):
+    parsed_globals.command = service
+    session.set_config_variable("region", region)
+    update_endpoint_url(session, parsed_globals)
+    assert parsed_globals.endpoint_url == (
+        f"https://{SSL_COMMON_NAMES[service][region]}"
+    )
+
+
+@pytest.mark.parametrize("service,region", parameters())
+def test_url_modified_from_event(parsed_globals, session, service, region):
+    assert parsed_globals.endpoint_url is None
+    parsed_globals.command = service
+    session.set_config_variable("region", region)
+    session.emit(
+        f"before-building-argument-table-parser.{service}",
+        args=[],
+        session=session,
+        argument_table={},
+        parsed_globals=parsed_globals,
+    )
+    assert parsed_globals.endpoint_url == (
+        f"https://{SSL_COMMON_NAMES[service][region]}"
+    )
+
+
+def test_dont_modify_provided_url(parsed_globals, session):
+    parsed_globals.endpoint_url = "http://test.com"
+    parsed_globals.command = "sqs"
+    update_endpoint_url(session, parsed_globals)
+    assert parsed_globals.endpoint_url == "http://test.com"