aws - sqs - fix queue url format via correct endpoint (cloud-custodia…

…n#7766)
newscorp-ghfb · Sep 20, 2022 · ca2a8e0 · ca2a8e0
1 parent 88423a7
commit ca2a8e0
Show file tree

Hide file tree

Showing 8 changed files with 132 additions and 8 deletions.
diff --git a/c7n/query.py b/c7n/query.py
@@ -68,8 +68,11 @@ def _invoke_client_enum(self, client, enum_op, params, path, retry=None):
     def filter(self, resource_manager, **params):
         """Query a set of resources."""
         m = self.resolve(resource_manager.resource_type)
-        client = local_session(self.session_factory).client(
-            m.service, resource_manager.config.region)
+        if resource_manager.get_client:
+            client = resource_manager.get_client()
+        else:
+            client = local_session(self.session_factory).client(
+                m.service, resource_manager.config.region)
         enum_op, path, extra_args = m.enum_spec
         if extra_args:
             params.update(extra_args)
@@ -443,6 +446,8 @@ class QueryResourceManager(ResourceManager, metaclass=QueryMeta):
 
     _generate_arn = None
 
+    get_client = None
+
     retry = staticmethod(
         get_retry((
             'TooManyRequestsException',

diff --git a/c7n/resources/sqs.py b/c7n/resources/sqs.py
@@ -23,7 +23,7 @@
 class DescribeQueue(DescribeSource):
 
     def augment(self, resources):
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
 
         def _augment(r):
             try:
@@ -82,6 +82,29 @@ class resource_type(TypeInfo):
         'config': QueueConfigSource
     }
 
+    def get_client(self):
+        # Work around the fact that boto picks a legacy endpoint by default
+        # which leads to queue urls pointing to legacy instead of standard
+        # which is at odds with config's resource id for the queues.
+        # additionally we need the standard endpoint to work with vpc endpoints.
+        #
+        # sqs canonoical endpoints
+        #  https://docs.aws.amazon.com/general/latest/gr/sqs-service.html
+        # boto3 bug
+        #  https://github.com/boto/botocore/issues/2683 - index of several other bugs
+        #  https://github.com/boto/boto3/issues/1900
+        #
+        # boto3 is transitioning to standard urls per https://github.com/boto/botocore/issues/2705
+        #
+        endpoint = 'https://sqs.{region}.amazonaws.com'.format(region=self.config.region)
+        # these only seem to have the legacy endpoints, so fall through to boto behavior.
+        if self.config.region in ('cn-north-1', 'cn-northwest-1'):
+            endpoint = None
+        params = {}
+        if endpoint:
+            params['endpoint_url'] = endpoint
+        return local_session(self.session_factory).client('sqs', **params)
+
     def get_permissions(self):
         perms = super(SQS, self).get_permissions()
         perms.append('sqs:GetQueueAttributes')
@@ -181,7 +204,7 @@ class RemovePolicyStatement(RemovePolicyBase):
 
     def process(self, resources):
         results = []
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
         for r in resources:
             try:
                 results += filter(None, [self.process_resource(client, r)])
@@ -240,7 +263,7 @@ class ModifyPolicyStatement(ModifyPolicyBase):
 
     def process(self, resources):
         results = []
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
         for r in resources:
             policy = json.loads(r.get('Policy') or '{}')
             policy_statements = policy.setdefault('Statement', [])
@@ -292,7 +315,7 @@ class DeleteSqsQueue(BaseAction):
     permissions = ('sqs:DeleteQueue',)
 
     def process(self, queues):
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
         for q in queues:
             self.process_queue(client, q)
 
@@ -333,7 +356,7 @@ def process(self, queues):
         session = local_session(self.manager.session_factory)
         key_id = session.client(
             'kms').describe_key(KeyId=key)['KeyMetadata']['KeyId']
-        client = session.client('sqs')
+        client = self.manager.get_client()
 
         for q in queues:
             self.process_queue(client, q, key_id)
@@ -376,7 +399,7 @@ class SetRetentionPeriod(BaseAction):
     permissions = ('sqs:SetQueueAttributes',)
 
     def process(self, queues):
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
         period = str(self.data.get('period', 345600))
         for q in queues:
             client.set_queue_attributes(

diff --git a/tests/data/placebo/test_sqs_endpoint_url/sqs.GetQueueAttributes_1.json b/tests/data/placebo/test_sqs_endpoint_url/sqs.GetQueueAttributes_1.json
@@ -0,0 +1,20 @@
+{
+    "status_code": 200,
+    "data": {
+        "Attributes": {
+            "QueueArn": "arn:aws:sqs:us-east-1:644160558196:devtest2",
+            "ApproximateNumberOfMessages": "0",
+            "ApproximateNumberOfMessagesNotVisible": "0",
+            "ApproximateNumberOfMessagesDelayed": "0",
+            "CreatedTimestamp": "1549200767",
+            "LastModifiedTimestamp": "1549200767",
+            "VisibilityTimeout": "30",
+            "MaximumMessageSize": "262144",
+            "MessageRetentionPeriod": "345600",
+            "DelaySeconds": "0",
+            "ReceiveMessageWaitTimeSeconds": "0",
+            "SqsManagedSseEnabled": "false"
+        },
+        "ResponseMetadata": {}
+    }
+}
diff --git a/tests/data/placebo/test_sqs_endpoint_url/sqs.GetQueueAttributes_2.json b/tests/data/placebo/test_sqs_endpoint_url/sqs.GetQueueAttributes_2.json
@@ -0,0 +1,20 @@
+{
+    "status_code": 200,
+    "data": {
+        "Attributes": {
+            "QueueArn": "arn:aws:sqs:us-east-1:644160558196:hubalytics-dev-ArchiveHourlyQueue-1VHR8KVX2MY48",
+            "ApproximateNumberOfMessages": "0",
+            "ApproximateNumberOfMessagesNotVisible": "0",
+            "ApproximateNumberOfMessagesDelayed": "0",
+            "CreatedTimestamp": "1579993745",
+            "LastModifiedTimestamp": "1579993745",
+            "VisibilityTimeout": "30",
+            "MaximumMessageSize": "262144",
+            "MessageRetentionPeriod": "345600",
+            "DelaySeconds": "0",
+            "ReceiveMessageWaitTimeSeconds": "0",
+            "SqsManagedSseEnabled": "false"
+        },
+        "ResponseMetadata": {}
+    }
+}
diff --git a/tests/data/placebo/test_sqs_endpoint_url/sqs.GetQueueAttributes_3.json b/tests/data/placebo/test_sqs_endpoint_url/sqs.GetQueueAttributes_3.json
@@ -0,0 +1,20 @@
+{
+    "status_code": 200,
+    "data": {
+        "Attributes": {
+            "QueueArn": "arn:aws:sqs:us-east-1:644160558196:maid-delivery",
+            "ApproximateNumberOfMessages": "0",
+            "ApproximateNumberOfMessagesNotVisible": "0",
+            "ApproximateNumberOfMessagesDelayed": "0",
+            "CreatedTimestamp": "1459708052",
+            "LastModifiedTimestamp": "1459708052",
+            "VisibilityTimeout": "30",
+            "MaximumMessageSize": "262144",
+            "MessageRetentionPeriod": "345600",
+            "DelaySeconds": "0",
+            "ReceiveMessageWaitTimeSeconds": "0",
+            "SqsManagedSseEnabled": "false"
+        },
+        "ResponseMetadata": {}
+    }
+}
diff --git a/tests/data/placebo/test_sqs_endpoint_url/sqs.ListQueues_1.json b/tests/data/placebo/test_sqs_endpoint_url/sqs.ListQueues_1.json
@@ -0,0 +1,11 @@
+{
+    "status_code": 200,
+    "data": {
+        "QueueUrls": [
+            "https://sqs.us-east-1.amazonaws.com/644160558196/devtest2",
+            "https://sqs.us-east-1.amazonaws.com/644160558196/hubalytics-dev-ArchiveHourlyQueue-1VHR8KVX2MY48",
+            "https://sqs.us-east-1.amazonaws.com/644160558196/maid-delivery"
+        ],
+        "ResponseMetadata": {}
+    }
+}
diff --git a/tests/data/placebo/test_sqs_endpoint_url/tagging.GetResources_1.json b/tests/data/placebo/test_sqs_endpoint_url/tagging.GetResources_1.json
@@ -0,0 +1,8 @@
+{
+    "status_code": 200,
+    "data": {
+        "PaginationToken": "",
+        "ResourceTagMappingList": [],
+        "ResponseMetadata": {}
+    }
+}
diff --git a/tests/test_sqs.py b/tests/test_sqs.py
@@ -17,6 +17,23 @@
 from c7n.resources.aws import shape_validate, Arn
 
 
+def test_sqs_endpoint_url(test):
+    session_factory = test.replay_flight_data(
+        "test_sqs_endpoint_url", region="us-east-1")
+    p = test.load_policy({
+        'name': 'sqs-check',
+        'resource': 'aws.sqs'
+    }, session_factory=session_factory)
+    urls = [q['QueueUrl'] for q in p.run()]
+    assert urls == [
+        'https://sqs.us-east-1.amazonaws.com/644160558196/devtest2',
+        'https://sqs.us-east-1.amazonaws.com/644160558196/hubalytics-dev-ArchiveHourlyQueue-1VHR8KVX2MY48', # noqa
+        'https://sqs.us-east-1.amazonaws.com/644160558196/maid-delivery',
+    ]
+    assert p.resource_manager.get_client().meta.endpoint_url == (
+        "https://sqs.us-east-1.amazonaws.com")
+
+
 def test_sqs_config_translate(test):
     # we're using a cwe event as a config, so have to mangle to
     # config's inane format (json strings in json)