From ce2d62ce36a078b692f21818c498d0d0fd14bdeb Mon Sep 17 00:00:00 2001 From: David Miller Date: Thu, 14 Jul 2022 17:28:58 -0400 Subject: [PATCH] Allow usage of SSL common name with deprecation. In the comming months, the method by which every AWS SDK resolves their endpoints, including aws-cli v1 and boto3, will be udpated with completely different behavior. At this time,the use of the `sslCommonName` paradigm will be fully deprecated. Additionally, it was mainly used to support python 2.6, which has not been supported by boto3 or the CLI since January 1, 2020. The boto team wanted to ease the transition by raising a warning to users and providing an optional environment variable to fully disable the behavior immediately. However, this causes breaking changes in v1 of aws-cli so additional changes were required in it to release the deprecation. This commit attempts to formally maintain the `sslCommonName` paradigm in aws-cli v1. This is accomplished by providing hardcoded, manual overrides for each service and region combination where this parameter is used over the standard `hostname`. The list of these service/region combinations was compiled by creating a client for every single combination and emitting the `endpoint_url` to a file twice. The first using a version of botocore that uses `sslCommonName` and one that doesn't, then diffing the two files to see which URLs changed. While behavior for the end user will largely be the same, if they do not disable the common name behavior by setting the environment variable `BOTO_DISABLE_COMMONNAME` to `true`, a warning will be omitted in the console once per session. Additional details can be found in https://github.com/boto/botocore/issues/2705. The override occurs when the `before-building-argument-table-parser` is omitted by an event handler. At this point, if the user has not set `endpoint_url` in `parsed_globals` (`--endpoint-url` flag from the command line) the endpoint that the CLI uses will be overridden to continue to use the SSL common name. There were a few different options of which event to run the override in, but ultimately this one was chosen because it was the least invasive to the existing code base. As this is very low level behavior for the CLI, minimizing the amount of changes made was prioritized over one that might seemt to make more logical sense like `top-level-args-parsed`, which is when the arguments passed to the command line are actually parsed through. However, the usage of a session instance was required in order to get config variables like region and partition. There is a strong possiblity that this customization override will run twice per call, but that is a design flaw the maintainers are aware of and have accpeted. Additionally for the `health` service, the region parameter needed to be overwritten because of it's use of pseudo regions like `aws-global`. In the other cases, the region within the URL essentially always matched the region, but since health uses the SSL common name in global pseudo regions, this change was required as well or an exception would be raised. --- .../enhancement-Endpoints-23278.json | 5 + awscli/clidriver.py | 2 +- awscli/customizations/commands.py | 2 +- .../customizations/overridesslcommonname.py | 138 ++++++++++++++++++ awscli/customizations/rds.py | 6 +- awscli/handlers.py | 2 + .../test_override_ssl_common_name.py | 64 ++++++++ .../test_overridesslcommonname.py | 75 ++++++++++ 8 files changed, 290 insertions(+), 4 deletions(-) create mode 100644 .changes/next-release/enhancement-Endpoints-23278.json create mode 100644 awscli/customizations/overridesslcommonname.py create mode 100644 tests/functional/test_override_ssl_common_name.py create mode 100644 tests/unit/customizations/test_overridesslcommonname.py diff --git a/.changes/next-release/enhancement-Endpoints-23278.json b/.changes/next-release/enhancement-Endpoints-23278.json new file mode 100644 index 000000000000..e8c8ac25cc04 --- /dev/null +++ b/.changes/next-release/enhancement-Endpoints-23278.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "Endpoints", + "description": "Enforce SSL common name as default endpoint url" +} diff --git a/awscli/clidriver.py b/awscli/clidriver.py index 304cebbe6be5..d7db5656c720 100644 --- a/awscli/clidriver.py +++ b/awscli/clidriver.py @@ -484,7 +484,7 @@ def __call__(self, args, parsed_globals): event = 'before-building-argument-table-parser.%s.%s' % \ (self._parent_name, self._name) self._emit(event, argument_table=self.arg_table, args=args, - session=self._session) + session=self._session, parsed_globals=parsed_globals) operation_parser = self._create_operation_parser(self.arg_table) self._add_help(operation_parser) parsed_args, remaining = operation_parser.parse_known_args(args) diff --git a/awscli/customizations/commands.py b/awscli/customizations/commands.py index 323b1cd6ab52..7c016751dbb7 100644 --- a/awscli/customizations/commands.py +++ b/awscli/customizations/commands.py @@ -134,7 +134,7 @@ def __call__(self, args, parsed_globals): event = 'before-building-argument-table-parser.%s' % \ ".".join(self.lineage_names) self._session.emit(event, argument_table=self._arg_table, args=args, - session=self._session) + session=self._session, parsed_globals=parsed_globals) parser = ArgTableArgParser(self.arg_table, self.subcommand_table) parsed_args, remaining = parser.parse_known_args(args) diff --git a/awscli/customizations/overridesslcommonname.py b/awscli/customizations/overridesslcommonname.py new file mode 100644 index 000000000000..f3dbd09fc829 --- /dev/null +++ b/awscli/customizations/overridesslcommonname.py @@ -0,0 +1,138 @@ +# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + + +SSL_COMMON_NAMES = { + "sqs": { + "af-south-1": "af-south-1.queue.amazonaws.com", + "ap-east-1": "ap-east-1.queue.amazonaws.com", + "ap-northeast-1": "ap-northeast-1.queue.amazonaws.com", + "ap-northeast-2": "ap-northeast-2.queue.amazonaws.com", + "ap-northeast-3": "ap-northeast-3.queue.amazonaws.com", + "ap-south-1": "ap-south-1.queue.amazonaws.com", + "ap-southeast-1": "ap-southeast-1.queue.amazonaws.com", + "ap-southeast-2": "ap-southeast-2.queue.amazonaws.com", + "ap-southeast-3": "ap-southeast-3.queue.amazonaws.com", + "ca-central-1": "ca-central-1.queue.amazonaws.com", + "eu-central-1": "eu-central-1.queue.amazonaws.com", + "eu-north-1": "eu-north-1.queue.amazonaws.com", + "eu-south-1": "eu-south-1.queue.amazonaws.com", + "eu-west-1": "eu-west-1.queue.amazonaws.com", + "eu-west-2": "eu-west-2.queue.amazonaws.com", + "eu-west-3": "eu-west-3.queue.amazonaws.com", + "me-south-1": "me-south-1.queue.amazonaws.com", + "sa-east-1": "sa-east-1.queue.amazonaws.com", + "us-east-1": "queue.amazonaws.com", + "us-east-2": "us-east-2.queue.amazonaws.com", + "us-west-1": "us-west-1.queue.amazonaws.com", + "us-west-2": "us-west-2.queue.amazonaws.com", + "cn-north-1": "cn-north-1.queue.amazonaws.com.cn", + "cn-northwest-1": "cn-northwest-1.queue.amazonaws.com.cn", + "us-gov-west-1": "us-gov-west-1.queue.amazonaws.com", + "us-isob-east-1": "us-isob-east-1.queue.sc2s.sgov.gov", + }, + "emr": { + "af-south-1": "af-south-1.elasticmapreduce.amazonaws.com", + "ap-east-1": "ap-east-1.elasticmapreduce.amazonaws.com", + "ap-northeast-1": "ap-northeast-1.elasticmapreduce.amazonaws.com", + "ap-northeast-2": "ap-northeast-2.elasticmapreduce.amazonaws.com", + "ap-northeast-3": "ap-northeast-3.elasticmapreduce.amazonaws.com", + "ap-south-1": "ap-south-1.elasticmapreduce.amazonaws.com", + "ap-southeast-1": "ap-southeast-1.elasticmapreduce.amazonaws.com", + "ap-southeast-2": "ap-southeast-2.elasticmapreduce.amazonaws.com", + "ap-southeast-3": "ap-southeast-3.elasticmapreduce.amazonaws.com", + "ca-central-1": "ca-central-1.elasticmapreduce.amazonaws.com", + "eu-north-1": "eu-north-1.elasticmapreduce.amazonaws.com", + "eu-south-1": "eu-south-1.elasticmapreduce.amazonaws.com", + "eu-west-1": "eu-west-1.elasticmapreduce.amazonaws.com", + "eu-west-2": "eu-west-2.elasticmapreduce.amazonaws.com", + "eu-west-3": "eu-west-3.elasticmapreduce.amazonaws.com", + "me-south-1": "me-south-1.elasticmapreduce.amazonaws.com", + "sa-east-1": "sa-east-1.elasticmapreduce.amazonaws.com", + "us-east-2": "us-east-2.elasticmapreduce.amazonaws.com", + "us-west-1": "us-west-1.elasticmapreduce.amazonaws.com", + "us-west-2": "us-west-2.elasticmapreduce.amazonaws.com", + }, + "rds": { + "us-east-1": "rds.amazonaws.com", + }, + "docdb": { + "us-east-1": "rds.amazonaws.com", + }, + "neptune": { + "us-east-1": "rds.amazonaws.com", + }, + "health": { + "aws-global": "health.us-east-1.amazonaws.com", + "af-south-1": "health.us-east-1.amazonaws.com", + "ap-east-1": "health.us-east-1.amazonaws.com", + "ap-northeast-1": "health.us-east-1.amazonaws.com", + "ap-northeast-2": "health.us-east-1.amazonaws.com", + "ap-northeast-3": "health.us-east-1.amazonaws.com", + "ap-south-1": "health.us-east-1.amazonaws.com", + "ap-southeast-1": "health.us-east-1.amazonaws.com", + "ap-southeast-2": "health.us-east-1.amazonaws.com", + "ap-southeast-3": "health.us-east-1.amazonaws.com", + "ca-central-1": "health.us-east-1.amazonaws.com", + "eu-central-1": "health.us-east-1.amazonaws.com", + "eu-north-1": "health.us-east-1.amazonaws.com", + "eu-south-1": "health.us-east-1.amazonaws.com", + "eu-west-1": "health.us-east-1.amazonaws.com", + "eu-west-2": "health.us-east-1.amazonaws.com", + "eu-west-3": "health.us-east-1.amazonaws.com", + "me-south-1": "health.us-east-1.amazonaws.com", + "sa-east-1": "health.us-east-1.amazonaws.com", + "us-east-1": "health.us-east-1.amazonaws.com", + "us-east-2": "health.us-east-1.amazonaws.com", + "us-west-1": "health.us-east-1.amazonaws.com", + "us-west-2": "health.us-east-1.amazonaws.com", + "cn-north-1": "health.cn-northwest-1.amazonaws.com.cn", + "cn-northwest-1": "health.cn-northwest-1.amazonaws.com.cn", + "aws-cn-global": "health.cn-northwest-1.amazonaws.com.cn", + }, +} + +REGION_TO_PARTITION_OVERRIDE = { + "aws-global": "aws", + "aws-cn-global": "aws-cn", +} + + +def register_override_ssl_common_name(cli): + cli.register_last( + "before-building-argument-table-parser", update_endpoint_url + ) + + +def update_endpoint_url(session, parsed_globals, **kwargs): + service = parsed_globals.command + endpoints = SSL_COMMON_NAMES.get(service) + region = session.get_config_variable("region") + # only change url if user has not overridden already themselves + if endpoints is not None and parsed_globals.endpoint_url is None: + endpoint_url = endpoints.get(region) + if endpoint_url is not None: + parsed_globals.endpoint_url = f"https://{endpoint_url}" + if service == "health": + _override_health_region(region, session, parsed_globals) + + +def _override_health_region(region, session, parsed_globals): + if region in REGION_TO_PARTITION_OVERRIDE: + partition = REGION_TO_PARTITION_OVERRIDE[region] + else: + partition = session.get_partition_for_region(region) + if partition == "aws-cn": + parsed_globals.region = "cn-northwest-1" + else: + parsed_globals.region = "us-east-1" diff --git a/awscli/customizations/rds.py b/awscli/customizations/rds.py index cac3173f3f76..5b62a79ef9c1 100644 --- a/awscli/customizations/rds.py +++ b/awscli/customizations/rds.py @@ -96,8 +96,10 @@ class GenerateDBAuthTokenCommand(BasicCommand): def _run_main(self, parsed_args, parsed_globals): rds = self._session.create_client( - 'rds', parsed_globals.region, parsed_globals.endpoint_url, - parsed_globals.verify_ssl + 'rds', + region_name=parsed_globals.region, + endpoint_url=parsed_globals.endpoint_url, + verify=parsed_globals.verify_ssl ) token = rds.generate_db_auth_token( DBHostname=parsed_args.hostname, diff --git a/awscli/handlers.py b/awscli/handlers.py index 0c1f8781886f..f0df63d47c7e 100644 --- a/awscli/handlers.py +++ b/awscli/handlers.py @@ -91,6 +91,7 @@ from awscli.customizations.sessionmanager import register_ssm_session from awscli.customizations.sms_voice import register_sms_voice_hide from awscli.customizations.dynamodb import register_dynamodb_paginator_fix +from awscli.customizations.overridesslcommonname import register_override_ssl_common_name def awscli_initialize(event_handlers): @@ -183,3 +184,4 @@ def awscli_initialize(event_handlers): register_ssm_session(event_handlers) register_sms_voice_hide(event_handlers) register_dynamodb_paginator_fix(event_handlers) + register_override_ssl_common_name(event_handlers) diff --git a/tests/functional/test_override_ssl_common_name.py b/tests/functional/test_override_ssl_common_name.py new file mode 100644 index 000000000000..1069e76c3a4a --- /dev/null +++ b/tests/functional/test_override_ssl_common_name.py @@ -0,0 +1,64 @@ +# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from awscli.testutils import BaseAWSCommandParamsTest +from awscli.customizations.overridesslcommonname import SSL_COMMON_NAMES + + +class OverrideSSLCommonNameTestCase(BaseAWSCommandParamsTest): + def _common_name_test_cases(self): + + service_ops = { + "sqs": "list-queues", + "emr": "list-clusters", + "rds": "describe-db-clusters", + "neptune": "describe-db-clusters", + "docdb": "describe-db-clusters", + } + for service, operation in service_ops.items(): + for region in SSL_COMMON_NAMES[service]: + yield (service, operation, region) + + def _assert_region_endpoint_used( + self, expected_region, expected_endpoint_url + ): + self.assertEqual( + expected_region, self.last_request_dict["context"]["client_region"] + ) + self.assertEqual( + expected_endpoint_url, + self.last_request_dict["url"], + ) + + def test_set_endpoint_url_arg(self): + for service, operation, region in self._common_name_test_cases(): + self.run_cmd(f"{service} {operation} --region {region}".split()) + expected_endpoint_url = ( + f"https://{SSL_COMMON_NAMES[service][region]}/" + ) + self._assert_region_endpoint_used(region, expected_endpoint_url) + + def test_override_health_endpoint_and_region(self): + expected_override_region = "us-east-1" + health_regions = SSL_COMMON_NAMES["health"] + for region in health_regions: + if region in ["cn-north-1", "cn-northwest-1", "aws-cn-global"]: + expected_override_region = "cn-northwest-1" + expected_endpoint_url = ( + f"https://{SSL_COMMON_NAMES['health'][region]}/" + ) + self.run_cmd(f"health describe-events --region {region}".split()) + self._assert_region_endpoint_used( + expected_override_region, + expected_endpoint_url, + ) diff --git a/tests/unit/customizations/test_overridesslcommonname.py b/tests/unit/customizations/test_overridesslcommonname.py new file mode 100644 index 000000000000..0a6e1978e846 --- /dev/null +++ b/tests/unit/customizations/test_overridesslcommonname.py @@ -0,0 +1,75 @@ +# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from awscli.testutils import create_clidriver +from awscli.customizations.overridesslcommonname import ( + update_endpoint_url, + SSL_COMMON_NAMES, +) + +import pytest +import argparse + + +def parameters(): + for service, regions in SSL_COMMON_NAMES.items(): + for region in regions: + yield (service, region) + + +@pytest.fixture +def parsed_globals(): + pg = argparse.Namespace() + pg.endpoint_url = None + pg.region = None + return pg + + +@pytest.fixture +def session(): + driver = create_clidriver() + return driver.session + + +@pytest.mark.parametrize("service,region", parameters()) +def test_update_endpoint_url(parsed_globals, session, service, region): + parsed_globals.command = service + session.set_config_variable("region", region) + update_endpoint_url(session, parsed_globals) + assert parsed_globals.endpoint_url == ( + f"https://{SSL_COMMON_NAMES[service][region]}" + ) + + +@pytest.mark.parametrize("service,region", parameters()) +def test_url_modified_from_event(parsed_globals, session, service, region): + assert parsed_globals.endpoint_url is None + parsed_globals.command = service + session.set_config_variable("region", region) + session.emit( + f"before-building-argument-table-parser.{service}", + args=[], + session=session, + argument_table={}, + parsed_globals=parsed_globals, + ) + assert parsed_globals.endpoint_url == ( + f"https://{SSL_COMMON_NAMES[service][region]}" + ) + + +def test_dont_modify_provided_url(parsed_globals, session): + parsed_globals.endpoint_url = "http://test.com" + parsed_globals.command = "sqs" + update_endpoint_url(session, parsed_globals) + assert parsed_globals.endpoint_url == "http://test.com"