From 1b43d1a67048221180d6128aa81b63c6cc9cc4df Mon Sep 17 00:00:00 2001 From: David Miller <45697098+dlm6693@users.noreply.github.com> Date: Tue, 24 Oct 2023 15:56:46 -0400 Subject: [PATCH] Add generic support for client context params (#3037) * add `client_context_params` config setting * add `client_context_params` to docs --- .../enhancement-Configuration-49047.json | 5 + botocore/args.py | 19 ++- botocore/config.py | 10 ++ botocore/docs/client.py | 54 +++++++ botocore/docs/service.py | 25 ++- tests/functional/docs/test_s3.py | 22 +++ tests/functional/test_context_params.py | 150 ++++++++++++++---- tests/unit/docs/test_client.py | 42 ++++- tests/unit/docs/test_service.py | 22 +++ 9 files changed, 311 insertions(+), 38 deletions(-) create mode 100644 .changes/next-release/enhancement-Configuration-49047.json diff --git a/.changes/next-release/enhancement-Configuration-49047.json b/.changes/next-release/enhancement-Configuration-49047.json new file mode 100644 index 0000000000..d640feb6e7 --- /dev/null +++ b/.changes/next-release/enhancement-Configuration-49047.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "Configuration", + "description": "Adds client context params support to ``Config``." +} diff --git a/botocore/args.py b/botocore/args.py index 83e6b1e27e..dbbcbe8a99 100644 --- a/botocore/args.py +++ b/botocore/args.py @@ -266,6 +266,7 @@ def compute_client_args( disable_request_compression=( client_config.disable_request_compression ), + client_context_params=client_config.client_context_params, ) self._compute_retry_config(config_kwargs) self._compute_connect_timeout(config_kwargs) @@ -641,14 +642,16 @@ def _build_endpoint_resolver( client_endpoint_url=endpoint_url, legacy_endpoint_url=endpoint.host, ) - # botocore does not support client context parameters generically - # for every service. Instead, the s3 config section entries are - # available as client context parameters. In the future, endpoint - # rulesets of services other than s3/s3control may require client - # context parameters. - client_context = ( - s3_config_raw if self._is_s3_service(service_name_raw) else {} - ) + # Client context params for s3 conflict with the available settings + # in the `s3` parameter on the `Config` object. If the same parameter + # is set in both places, the value in the `s3` parameter takes priority. + if client_config is not None: + client_context = client_config.client_context_params or {} + else: + client_context = {} + if self._is_s3_service(service_name_raw): + client_context.update(s3_config_raw) + sig_version = ( client_config.signature_version if client_config is not None diff --git a/botocore/config.py b/botocore/config.py index fe968911ba..5ae7521530 100644 --- a/botocore/config.py +++ b/botocore/config.py @@ -219,6 +219,15 @@ class Config: :param disable_request_compression: Disables request body compression if set to True. + Defaults to None. + + :type client_context_params: dict + :param client_context_params: A dictionary of parameters specific to + individual services. If available, valid parameters can be found in + the ``Client Context Parameters`` section of the service client's + documentation. Invalid parameters or ones that are not used by the + specified service will be ignored. + Defaults to None. """ @@ -247,6 +256,7 @@ class Config: ('tcp_keepalive', None), ('request_min_compression_size_bytes', None), ('disable_request_compression', None), + ('client_context_params', None), ] ) diff --git a/botocore/docs/client.py b/botocore/docs/client.py index 9559594697..bc9b2658c9 100644 --- a/botocore/docs/client.py +++ b/botocore/docs/client.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. import os +from botocore import xform_name from botocore.compat import OrderedDict from botocore.docs.bcdoc.restdoc import DocumentStructure from botocore.docs.example import ResponseExampleDocumenter @@ -399,3 +400,56 @@ def _add_response_params(self, section, shape): shape, include=[self._GENERIC_ERROR_SHAPE], ) + + +class ClientContextParamsDocumenter: + _CONFIG_GUIDE_LINK = ( + 'https://boto3.amazonaws.com/' + 'v1/documentation/api/latest/guide/configuration.html' + ) + + OMITTED_CONTEXT_PARAMS = { + 's3': ( + 'Accelerate', + 'DisableMultiRegionAccessPoints', + 'ForcePathStyle', + 'UseArnRegion', + ), + 's3control': ('UseArnRegion',), + } + + def __init__(self, service_name, context_params): + self._service_name = service_name + self._context_params = context_params + + def document_context_params(self, section): + self._add_title(section) + self._add_overview(section) + self._add_context_params_list(section) + + def _add_title(self, section): + section.style.h2('Client Context Parameters') + + def _add_overview(self, section): + section.style.new_line() + section.write( + 'Client context parameters are configurable on a client ' + 'instance via the ``client_context_params`` parameter in the ' + '``Config`` object. For more detailed instructions and examples ' + 'on the exact usage of context params see the ' + ) + section.style.external_link( + title='configuration guide', + link=self._CONFIG_GUIDE_LINK, + ) + section.write('.') + section.style.new_line() + + def _add_context_params_list(self, section): + section.style.new_line() + sn = f'``{self._service_name}``' + section.writeln(f'The available {sn} client context params are:') + for param in self._context_params: + section.style.new_line() + name = f'``{xform_name(param.name)}``' + section.write(f'* {name} ({param.type}) - {param.documentation}') diff --git a/botocore/docs/service.py b/botocore/docs/service.py index fa183bc831..d20a889dc9 100644 --- a/botocore/docs/service.py +++ b/botocore/docs/service.py @@ -11,7 +11,11 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from botocore.docs.bcdoc.restdoc import DocumentStructure -from botocore.docs.client import ClientDocumenter, ClientExceptionsDocumenter +from botocore.docs.client import ( + ClientContextParamsDocumenter, + ClientDocumenter, + ClientExceptionsDocumenter, +) from botocore.docs.paginator import PaginatorDocumenter from botocore.docs.waiter import WaiterDocumenter from botocore.exceptions import DataNotFoundError @@ -37,6 +41,7 @@ def __init__(self, service_name, session, root_docs_path): 'client-exceptions', 'paginator-api', 'waiter-api', + 'client-context-params', ] def document_service(self): @@ -52,6 +57,10 @@ def document_service(self): self.client_exceptions(doc_structure.get_section('client-exceptions')) self.paginator_api(doc_structure.get_section('paginator-api')) self.waiter_api(doc_structure.get_section('waiter-api')) + context_params_section = doc_structure.get_section( + 'client-context-params' + ) + self.client_context_params(context_params_section) return doc_structure.flush_structure() def title(self, section): @@ -108,3 +117,17 @@ def get_examples(self, service_name, api_version=None): service_name, 'examples-1', api_version ) return examples['examples'] + + def client_context_params(self, section): + omitted_params = ClientContextParamsDocumenter.OMITTED_CONTEXT_PARAMS + params_to_omit = omitted_params.get(self._service_name, []) + service_model = self._client.meta.service_model + raw_context_params = service_model.client_context_parameters + context_params = [ + p for p in raw_context_params if p.name not in params_to_omit + ] + if context_params: + context_param_documenter = ClientContextParamsDocumenter( + self._service_name, context_params + ) + context_param_documenter.document_context_params(section) diff --git a/tests/functional/docs/test_s3.py b/tests/functional/docs/test_s3.py index 0caf27f041..fc9f125b6f 100644 --- a/tests/functional/docs/test_s3.py +++ b/tests/functional/docs/test_s3.py @@ -10,6 +10,8 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +from botocore import xform_name +from botocore.docs.client import ClientContextParamsDocumenter from botocore.docs.service import ServiceDocumenter from tests.functional.docs import BaseDocsFunctionalTest @@ -78,3 +80,23 @@ def test_copy_source_param_docs_also_modified(self): self.assert_contains_line( "You can also provide this value as a dictionary", param_docs ) + + def test_s3_context_params_omitted(self): + omitted_params = ClientContextParamsDocumenter.OMITTED_CONTEXT_PARAMS + s3_omitted_params = omitted_params['s3'] + content = ServiceDocumenter( + 's3', self._session, self.root_services_path + ).document_service() + for param in s3_omitted_params: + param_name = f'``{xform_name(param)}``' + self.assert_not_contains_line(param_name, content) + + def test_s3control_context_params_omitted(self): + omitted_params = ClientContextParamsDocumenter.OMITTED_CONTEXT_PARAMS + s3control_omitted_params = omitted_params['s3control'] + content = ServiceDocumenter( + 's3control', self._session, self.root_services_path + ).document_service() + for param in s3control_omitted_params: + param_name = f'``{xform_name(param)}``' + self.assert_not_contains_line(param_name, content) diff --git a/tests/functional/test_context_params.py b/tests/functional/test_context_params.py index 03eb9ae965..766eea32f5 100644 --- a/tests/functional/test_context_params.py +++ b/tests/functional/test_context_params.py @@ -59,6 +59,11 @@ "documentation": "", "type": "String", }, + "BarClientContextParamName": { + "required": False, + "documentation": "", + "type": "String", + }, }, } @@ -134,7 +139,11 @@ "FooClientContextParamName": { "documentation": "My mock client context parameter", "type": "string", - } + }, + "BarClientContextParamName": { + "documentation": "My mock client context parameter", + "type": "string", + }, }, } @@ -224,86 +233,178 @@ **FAKE_MODEL_WITH_CLIENT_CONTEXT_PARAM, "metadata": S3CONTROL_METADATA, } +CLIENT_CONTEXT_PARAM_INPUT = { + "foo_client_context_param_name": "foo_context_param_value" +} +OTHER_CLIENT_CONTEXT_PARAM_INPUT = { + "bar_client_context_param_name": "bar_value" +} + +CONFIG_WITH_S3 = Config(s3=CLIENT_CONTEXT_PARAM_INPUT) +CONFIG_WITH_CLIENT_CONTEXT_PARAMS = Config( + client_context_params=CLIENT_CONTEXT_PARAM_INPUT +) +CONFIG_WITH_S3_AND_CLIENT_CONTEXT_PARAMS = Config( + s3=CLIENT_CONTEXT_PARAM_INPUT, + client_context_params=OTHER_CLIENT_CONTEXT_PARAM_INPUT, +) +CONFIG_WITH_CONFLICTING_S3_AND_CLIENT_CONTEXT_PARAMS = Config( + s3=CLIENT_CONTEXT_PARAM_INPUT, + client_context_params={"foo_client_context_param_name": "bar_value"}, +) +NO_CTX_PARAM_EXPECTED_CALL_KWARGS = {"Region": "us-east-1"} +CTX_PARAM_EXPECTED_CALL_KWARGS = { + **NO_CTX_PARAM_EXPECTED_CALL_KWARGS, + "FooClientContextParamName": "foo_context_param_value", +} +MULTIPLE_CONTEXT_PARAMS_EXPECTED_CALL_KWARGS = { + **CTX_PARAM_EXPECTED_CALL_KWARGS, + "BarClientContextParamName": "bar_value", +} @pytest.mark.parametrize( - 'service_name,service_model,ruleset,call_should_include_ctx_param', + 'service_name,service_model,ruleset,config,expected_call_kwargs', [ # s3 ( 's3', FAKE_S3_MODEL_WITH_CLIENT_CONTEXT_PARAM, FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, - True, + CONFIG_WITH_S3, + CTX_PARAM_EXPECTED_CALL_KWARGS, ), ( 's3', FAKE_S3_MODEL_WITH_CLIENT_CONTEXT_PARAM, FAKE_RULESET_WITHOUT_ANY_CONTEXT_PARAMS, - False, + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, ), ( 's3', FAKE_S3_MODEL_WITHOUT_ANY_CONTEXT_PARAMS, FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, - False, + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, ), ( 's3', FAKE_S3_MODEL_WITHOUT_ANY_CONTEXT_PARAMS, FAKE_RULESET_WITHOUT_ANY_CONTEXT_PARAMS, - False, + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, + ), + ( + 's3', + FAKE_S3_MODEL_WITH_CLIENT_CONTEXT_PARAM, + FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, + CONFIG_WITH_CLIENT_CONTEXT_PARAMS, + CTX_PARAM_EXPECTED_CALL_KWARGS, + ), + # use both s3 and client_context_params when they don't overlap + ( + 's3', + FAKE_S3_MODEL_WITH_CLIENT_CONTEXT_PARAM, + FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, + CONFIG_WITH_S3_AND_CLIENT_CONTEXT_PARAMS, + MULTIPLE_CONTEXT_PARAMS_EXPECTED_CALL_KWARGS, + ), + # use s3 over client_context_params when they overlap + ( + 's3', + FAKE_S3_MODEL_WITH_CLIENT_CONTEXT_PARAM, + FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, + CONFIG_WITH_CONFLICTING_S3_AND_CLIENT_CONTEXT_PARAMS, + CTX_PARAM_EXPECTED_CALL_KWARGS, ), # s3control ( 's3control', FAKE_S3CONTROL_MODEL_WITH_CLIENT_CONTEXT_PARAM, FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, - True, + CONFIG_WITH_S3, + CTX_PARAM_EXPECTED_CALL_KWARGS, ), ( 's3control', FAKE_S3CONTROL_MODEL_WITH_CLIENT_CONTEXT_PARAM, FAKE_RULESET_WITHOUT_ANY_CONTEXT_PARAMS, - False, + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, ), ( 's3control', FAKE_S3CONTROL_MODEL_WITHOUT_ANY_CONTEXT_PARAMS, FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, - False, + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, ), ( 's3control', FAKE_S3CONTROL_MODEL_WITHOUT_ANY_CONTEXT_PARAMS, FAKE_RULESET_WITHOUT_ANY_CONTEXT_PARAMS, - False, + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, + ), + ( + 's3control', + FAKE_S3CONTROL_MODEL_WITH_CLIENT_CONTEXT_PARAM, + FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, + CONFIG_WITH_CLIENT_CONTEXT_PARAMS, + CTX_PARAM_EXPECTED_CALL_KWARGS, + ), + # use both s3 and client_context_params when they don't overlap + ( + 's3control', + FAKE_S3CONTROL_MODEL_WITH_CLIENT_CONTEXT_PARAM, + FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, + CONFIG_WITH_S3_AND_CLIENT_CONTEXT_PARAMS, + MULTIPLE_CONTEXT_PARAMS_EXPECTED_CALL_KWARGS, + ), + # use s3 over client_context_params when they overlap + ( + 's3control', + FAKE_S3CONTROL_MODEL_WITH_CLIENT_CONTEXT_PARAM, + FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, + CONFIG_WITH_CONFLICTING_S3_AND_CLIENT_CONTEXT_PARAMS, + CTX_PARAM_EXPECTED_CALL_KWARGS, ), - # botocore does not currently support client context params for - # services other than s3 and s3-control. + # otherservice ( 'otherservice', FAKE_MODEL_WITH_CLIENT_CONTEXT_PARAM, FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, - False, # would be True for s3 and s3control + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, ), ( 'otherservice', FAKE_MODEL_WITH_CLIENT_CONTEXT_PARAM, FAKE_RULESET_WITHOUT_ANY_CONTEXT_PARAMS, - False, # same as for s3 and s3control + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, ), ( 'otherservice', FAKE_MODEL_WITHOUT_ANY_CONTEXT_PARAMS, FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, - False, # same as for s3 and s3control + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, ), ( 'otherservice', FAKE_MODEL_WITHOUT_ANY_CONTEXT_PARAMS, FAKE_RULESET_WITHOUT_ANY_CONTEXT_PARAMS, - False, # same as for s3 and s3control + CONFIG_WITH_S3, + NO_CTX_PARAM_EXPECTED_CALL_KWARGS, + ), + ( + 'otherservice', + FAKE_MODEL_WITH_CLIENT_CONTEXT_PARAM, + FAKE_RULESET_WITH_CLIENT_CONTEXT_PARAM, + CONFIG_WITH_CLIENT_CONTEXT_PARAMS, + CTX_PARAM_EXPECTED_CALL_KWARGS, ), ], ) @@ -313,7 +414,8 @@ def test_client_context_param_sent_to_endpoint_resolver( service_name, service_model, ruleset, - call_should_include_ctx_param, + config, + expected_call_kwargs, ): # patch loader to return fake service model and fake endpoint ruleset patch_load_service_model( @@ -321,13 +423,11 @@ def test_client_context_param_sent_to_endpoint_resolver( ) # construct client using patched loader and a config object with an s3 - # section that sets the foo_context_param to a value + # or client_context_param section that sets the foo_context_param to a value client = patched_session.create_client( service_name, region_name='us-east-1', - config=Config( - s3={'foo_client_context_param_name': 'foo_context_param_value'} - ), + config=config, ) # Stub client to prevent a request from getting sent and ascertain that @@ -342,13 +442,7 @@ def test_client_context_param_sent_to_endpoint_resolver( ) as mock_resolve_endpoint: client.mock_operation(MockOpParam='mock-op-param-value') - if call_should_include_ctx_param: - mock_resolve_endpoint.assert_called_once_with( - Region='us-east-1', - FooClientContextParamName='foo_context_param_value', - ) - else: - mock_resolve_endpoint.assert_called_once_with(Region='us-east-1') + mock_resolve_endpoint.assert_called_once_with(**expected_call_kwargs) @pytest.mark.parametrize( diff --git a/tests/unit/docs/test_client.py b/tests/unit/docs/test_client.py index a0ec02a386..7785b7c806 100644 --- a/tests/unit/docs/test_client.py +++ b/tests/unit/docs/test_client.py @@ -10,7 +10,11 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from botocore.docs.client import ClientDocumenter, ClientExceptionsDocumenter +from botocore.docs.client import ( + ClientContextParamsDocumenter, + ClientDocumenter, + ClientExceptionsDocumenter, +) from tests.unit.docs import BaseDocsTest @@ -167,3 +171,39 @@ def test_modeled_exceptions(self): 'myservice', 'client/exceptions', 'SomeException' ), ) + + +class TestClientContextParamsDocumenter(BaseDocsTest): + def setUp(self): + super().setUp() + self.json_model['clientContextParams'] = { + 'ClientContextParam1': { + 'type': 'string', + 'documentation': 'A client context param', + }, + 'ClientContextParam2': { + 'type': 'boolean', + 'documentation': 'A second client context param', + }, + } + self.setup_client() + service_model = self.client.meta.service_model + self.context_params_documenter = ClientContextParamsDocumenter( + service_model.service_name, service_model.client_context_parameters + ) + + def test_client_context_params(self): + self.context_params_documenter.document_context_params( + self.doc_structure + ) + self.assert_contains_lines_in_order( + [ + '========================', + 'Client Context Parameters', + '========================', + 'Client context parameters are configurable', + 'The available ``myservice`` client context params are:', + '* ``client_context_param1`` (string) - A client context param', + '* ``client_context_param2`` (boolean) - A second client context param', + ] + ) diff --git a/tests/unit/docs/test_service.py b/tests/unit/docs/test_service.py index ebdb1d4357..01040ae2fe 100644 --- a/tests/unit/docs/test_service.py +++ b/tests/unit/docs/test_service.py @@ -21,6 +21,9 @@ class TestServiceDocumenter(BaseDocsTest): def setUp(self): super().setUp() + self.setup_documenter() + + def setup_documenter(self): self.add_shape_to_params('Biz', 'String') self.setup_client() with mock.patch( @@ -97,3 +100,22 @@ def test_document_service_no_waiter(self): os.remove(self.waiter_model_file) contents = self.service_documenter.document_service().decode('utf-8') self.assertNotIn('Waiters', contents) + + def test_document_service_no_context_params(self): + contents = self.service_documenter.document_service().decode('utf-8') + self.assertNotIn('Client Context Parameters', contents) + + def test_document_service_context_params(self): + self.json_model['clientContextParams'] = { + 'ClientContextParam1': { + 'type': 'string', + 'documentation': 'A client context param', + }, + 'ClientContextParam2': { + 'type': 'boolean', + 'documentation': 'A second client context param', + }, + } + self.setup_documenter() + contents = self.service_documenter.document_service().decode('utf-8') + self.assertIn('Client Context Parameters', contents)