From 263c83f15b960f7159e4746c1c26ff72116a85f3 Mon Sep 17 00:00:00 2001 From: Alex Shovlin Date: Thu, 19 Dec 2024 14:59:42 -0500 Subject: [PATCH 1/2] Add support for stringArray and operationContextParams This allows the CLI's endpoint resolver to consider collections of strings as an input to endpoint resolution, and use operationContextParams to resolve parameters from the operation's input. --- awscli/botocore/endpoint_provider.py | 14 +++--- awscli/botocore/model.py | 4 ++ awscli/botocore/regions.py | 17 +++++++ awscli/botocore/utils.py | 29 ++++++++++++ .../endpoints/test-cases/array-index.json | 37 +++++++++++++++ .../endpoints/valid-rules/array-index.json | 47 +++++++++++++++++++ tests/unit/botocore/test_endpoint_provider.py | 31 ++++++++++++ tests/unit/botocore/test_utils.py | 39 +++++++++++++++ 8 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 tests/unit/botocore/data/endpoints/test-cases/array-index.json create mode 100644 tests/unit/botocore/data/endpoints/valid-rules/array-index.json diff --git a/awscli/botocore/endpoint_provider.py b/awscli/botocore/endpoint_provider.py index 83a7b1291b20..cf1d8d7288ac 100644 --- a/awscli/botocore/endpoint_provider.py +++ b/awscli/botocore/endpoint_provider.py @@ -24,7 +24,6 @@ import logging import re from enum import Enum -from functools import lru_cache from string import Formatter from typing import NamedTuple @@ -36,6 +35,7 @@ InvalidArnException, is_valid_ipv4_endpoint_url, is_valid_ipv6_endpoint_url, + lru_cache_weakref, normalize_url_path, percent_encode, ) @@ -43,7 +43,7 @@ logger = logging.getLogger(__name__) TEMPLATE_STRING_RE = re.compile(r"\{[a-zA-Z#]+\}") -GET_ATTR_RE = re.compile(r"(\w+)\[(\d+)\]") +GET_ATTR_RE = re.compile(r"(\w*)\[(\d+)\]") VALID_HOST_LABEL_RE = re.compile( r"^(?!-)[a-zA-Z\d-]{1,63}(?= len(value): return None return value[index] @@ -577,6 +578,7 @@ class ParameterType(Enum): string = str boolean = bool + stringarray = tuple class ParameterDefinition: @@ -600,7 +602,7 @@ def __init__( except AttributeError: raise EndpointResolutionError( msg=f"Unknown parameter type: {parameter_type}. " - "A parameter must be of type string or boolean." + "A parameter must be of type string, boolean, or stringarray." ) self.documentation = documentation self.builtin = builtIn @@ -703,7 +705,7 @@ class EndpointProvider: def __init__(self, ruleset_data, partition_data): self.ruleset = RuleSet(**ruleset_data, partitions=partition_data) - @lru_cache(maxsize=CACHE_SIZE) + @lru_cache_weakref(maxsize=CACHE_SIZE) def resolve_endpoint(self, **input_parameters): """Match input parameters to a rule. diff --git a/awscli/botocore/model.py b/awscli/botocore/model.py index 0a90ca2d80ff..29e434a253fd 100644 --- a/awscli/botocore/model.py +++ b/awscli/botocore/model.py @@ -588,6 +588,10 @@ def context_parameters(self): and 'name' in shape.metadata['contextParam'] ] + @CachedProperty + def operation_context_parameters(self): + return self._operation_model.get('operationContextParams', []) + @CachedProperty def request_compression(self): return self._operation_model.get('requestcompression') diff --git a/awscli/botocore/regions.py b/awscli/botocore/regions.py index ead1c3ce8418..1e17bdeddf91 100644 --- a/awscli/botocore/regions.py +++ b/awscli/botocore/regions.py @@ -21,6 +21,8 @@ import re from enum import Enum +import jmespath + from botocore import UNSIGNED, xform_name from botocore.auth import AUTH_TYPE_MAPS from botocore.endpoint_provider import EndpointProvider @@ -530,6 +532,13 @@ def _resolve_param_from_context( ) if dynamic is not None: return dynamic + operation_context_params = ( + self._resolve_param_as_operation_context_param( + param_name, operation_model, call_args + ) + ) + if operation_context_params is not None: + return operation_context_params return self._resolve_param_as_client_context_param(param_name) def _resolve_param_as_static_context_param( @@ -552,6 +561,14 @@ def _resolve_param_as_client_context_param(self, param_name): client_ctx_varname = client_ctx_params[param_name] return self._client_context.get(client_ctx_varname) + def _resolve_param_as_operation_context_param( + self, param_name, operation_model, call_args + ): + operation_ctx_params = operation_model.operation_context_parameters + if param_name in operation_ctx_params: + path = operation_ctx_params[param_name]['path'] + return jmespath.search(path, call_args) + def _resolve_param_as_builtin(self, builtin_name, builtins): if builtin_name not in EndpointResolverBuiltins.__members__.values(): raise UnknownEndpointResolutionBuiltInName(name=builtin_name) diff --git a/awscli/botocore/utils.py b/awscli/botocore/utils.py index 13eb55b4d4d3..1d13a7d98b53 100644 --- a/awscli/botocore/utils.py +++ b/awscli/botocore/utils.py @@ -1263,6 +1263,35 @@ def _cache_guard(self, *args, **kwargs): return _cache_guard +def lru_cache_weakref(*cache_args, **cache_kwargs): + """ + Version of functools.lru_cache that stores a weak reference to ``self``. + Serves the same purpose as :py:func:`instance_cache` but uses Python's + functools implementation which offers ``max_size`` and ``typed`` properties. + lru_cache is a global cache even when used on a method. The cache's + reference to ``self`` will prevent garbage collection of the object. This + wrapper around functools.lru_cache replaces the reference to ``self`` with + a weak reference to not interfere with garbage collection. + """ + + def wrapper(func): + @functools.lru_cache(*cache_args, **cache_kwargs) + def func_with_weakref(weakref_to_self, *args, **kwargs): + return func(weakref_to_self(), *args, **kwargs) + + @functools.wraps(func) + def inner(self, *args, **kwargs): + for kwarg_key, kwarg_value in kwargs.items(): + if isinstance(kwarg_value, list): + kwargs[kwarg_key] = tuple(kwarg_value) + return func_with_weakref(weakref.ref(self), *args, **kwargs) + + inner.cache_info = func_with_weakref.cache_info + return inner + + return wrapper + + def switch_host_s3_accelerate(request, operation_name, **kwargs): """Switches the current s3 endpoint with an S3 Accelerate endpoint""" diff --git a/tests/unit/botocore/data/endpoints/test-cases/array-index.json b/tests/unit/botocore/data/endpoints/test-cases/array-index.json new file mode 100644 index 000000000000..aa73d460d33e --- /dev/null +++ b/tests/unit/botocore/data/endpoints/test-cases/array-index.json @@ -0,0 +1,37 @@ +{ + "version": "1.0", + "testCases": [ + { + "documentation": "Access an array index at index 0", + "params": { + "ResourceList": ["resource"] + }, + "expect": { + "endpoint": { + "url": "https://www.resource.example.com" + } + } + }, + { + "documentation": "Resolved value when array is explictly set to empty", + "params": { + "ResourceList": [] + }, + "expect": { + "endpoint": { + "url": "https://www.example.com" + } + } + }, + { + "documentation": "Resolved value to default if array is unset", + "params": { + }, + "expect": { + "endpoint": { + "url": "https://www.default1.example.com" + } + } + } + ] +} diff --git a/tests/unit/botocore/data/endpoints/valid-rules/array-index.json b/tests/unit/botocore/data/endpoints/valid-rules/array-index.json new file mode 100644 index 000000000000..3d426b09aa64 --- /dev/null +++ b/tests/unit/botocore/data/endpoints/valid-rules/array-index.json @@ -0,0 +1,47 @@ +{ + "version": "1.3", + "parameters": { + "ResourceList": { + "required": true, + "default": ["default1", "default2"], + "type": "stringArray" + } + }, + "rules": [ + { + "documentation": "Array is set, retrieve index 0", + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "ResourceList" + } + ] + }, + { + "fn": "getAttr", + "argv": [ + { + "ref": "ResourceList" + }, + "[0]" + ], + "assign": "resourceid" + } + ], + "endpoint": { + "url": "https://www.{resourceid}.example.com" + }, + "type": "endpoint" + }, + { + "documentation": "Fallback when array is unset", + "conditions": [], + "endpoint": { + "url": "https://www.example.com" + }, + "type": "endpoint" + } + ] +} diff --git a/tests/unit/botocore/test_endpoint_provider.py b/tests/unit/botocore/test_endpoint_provider.py index 2c1cdbb88a77..4137e86ead12 100644 --- a/tests/unit/botocore/test_endpoint_provider.py +++ b/tests/unit/botocore/test_endpoint_provider.py @@ -134,6 +134,7 @@ def endpoint_rule(): def ruleset_testcases(): filenames = [ + "array-index", "aws-region", "default-values", "eventbridge", @@ -457,3 +458,33 @@ def test_auth_schemes_conversion_first_authtype_unknown( at, sc = empty_resolver.auth_schemes_to_signing_ctx(auth_schemes) assert at == 'bar' assert sc == {'region': 'ap-south-2', 'signing_name': 's3'} + + +@pytest.mark.parametrize( + "value, path, expected_value", + [ + ({"foo": ['bar']}, 'baz[0]', None), # Missing index + ({"foo": ['bar']}, 'foo[1]', None), # Out of range index + ({"foo": ['bar']}, 'foo[0]', "bar"), # Named index + (("foo",), '[0]', "foo"), # Bare index + ({"foo": {}}, 'foo.bar[0]', None), # Missing index from split path + ( + {"foo": {'bar': []}}, + 'foo.bar[0]', + None, + ), # Out of range from split path + ( + {"foo": {"bar": "baz"}}, + 'foo.bar', + "baz", + ), # Split path with named index + ( + {"foo": {"bar": ["baz"]}}, + 'foo.bar[0]', + "baz", + ), # Split path with numeric index + ], +) +def test_get_attr(rule_lib, value, path, expected_value): + result = rule_lib.get_attr(value, path) + assert result == expected_value diff --git a/tests/unit/botocore/test_utils.py b/tests/unit/botocore/test_utils.py index 7c16e520b7e1..e457ea01f20d 100644 --- a/tests/unit/botocore/test_utils.py +++ b/tests/unit/botocore/test_utils.py @@ -24,6 +24,7 @@ import datetime import copy import operator +from sys import getrefcount import botocore from botocore import xform_name @@ -70,6 +71,7 @@ from botocore.utils import instance_cache from botocore.utils import merge_dicts from botocore.utils import lowercase_dict +from botocore.utils import lru_cache_weakref from botocore.utils import get_service_module_name from botocore.utils import get_encoding_from_headers from botocore.utils import percent_encode_sequence @@ -3619,3 +3621,40 @@ def test_is_s3_accelerate_url(url, expected): def test_get_encoding_from_headers(headers, default, expected): charset = get_encoding_from_headers(HeadersDict(headers), default=default) assert charset == expected + + +def test_lru_cache_weakref(): + class ClassWithCachedMethod: + @lru_cache_weakref(maxsize=10) + def cached_fn(self, a, b): + return a + b + + cls1 = ClassWithCachedMethod() + cls2 = ClassWithCachedMethod() + + assert cls1.cached_fn.cache_info().currsize == 0 + assert getrefcount(cls1) == 2 + assert getrefcount(cls2) == 2 + # "The count returned is generally one higher than you might expect, because + # it includes the (temporary) reference as an argument to getrefcount()." + # https://docs.python.org/3.8/library/sys.html#getrefcount + + cls1.cached_fn(1, 1) + cls2.cached_fn(1, 1) + + # The cache now has two entries, but the reference count remains the same as + # before. + assert cls1.cached_fn.cache_info().currsize == 2 + assert getrefcount(cls1) == 2 + assert getrefcount(cls2) == 2 + + # Deleting one of the objects does not interfere with the cache entries + # related to the other object. + del cls1 + assert cls2.cached_fn.cache_info().currsize == 2 + assert cls2.cached_fn.cache_info().hits == 0 + assert cls2.cached_fn.cache_info().misses == 2 + cls2.cached_fn(1, 1) + assert cls2.cached_fn.cache_info().currsize == 2 + assert cls2.cached_fn.cache_info().hits == 1 # the call was a cache hit + assert cls2.cached_fn.cache_info().misses == 2 From 838d7c019841c515c0ac3788c124368be7a25f55 Mon Sep 17 00:00:00 2001 From: Alex Shovlin Date: Fri, 20 Dec 2024 11:28:56 -0500 Subject: [PATCH 2/2] Add changelog entry --- .changes/next-release/enhancement-endpoints-85370.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/next-release/enhancement-endpoints-85370.json diff --git a/.changes/next-release/enhancement-endpoints-85370.json b/.changes/next-release/enhancement-endpoints-85370.json new file mode 100644 index 000000000000..27757334b857 --- /dev/null +++ b/.changes/next-release/enhancement-endpoints-85370.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "endpoints", + "description": "Add support for ``stringArray`` parameters and the ``operationContextParams`` trait when resolving service endpoints." +}