Merge remote-tracking branch 'upstream/master' into upstream-changes

Dockerfile

@@ -1 +1 @@
-docker/cli
+docker/c7n

c7n/query.py

@@ -68,8 +68,11 @@ def _invoke_client_enum(self, client, enum_op, params, path, retry=None):
     def filter(self, resource_manager, **params):
         """Query a set of resources."""
         m = self.resolve(resource_manager.resource_type)
-        client = local_session(self.session_factory).client(
-            m.service, resource_manager.config.region)
+        if resource_manager.get_client:
+            client = resource_manager.get_client()
+        else:
+            client = local_session(self.session_factory).client(
+                m.service, resource_manager.config.region)
         enum_op, path, extra_args = m.enum_spec
         if extra_args:
             params.update(extra_args)
@@ -443,6 +446,8 @@ class QueryResourceManager(ResourceManager, metaclass=QueryMeta):
 
     _generate_arn = None
 
+    get_client = None
+
     retry = staticmethod(
         get_retry((
             'TooManyRequestsException',

c7n/resources/elasticsearch.py

@@ -12,6 +12,7 @@
 from c7n.utils import chunks, local_session, type_schema
 from c7n.tags import Tag, RemoveTag, TagActionFilter, TagDelayedAction
 from c7n.filters.kms import KmsRelatedFilter
+import c7n.filters.policystatement as polstmt_filter
 
 from .securityhub import PostFinding
 
@@ -209,6 +210,20 @@ def process(self, resources, event=None):
         return results
 
 
+@ElasticSearchDomain.filter_registry.register('has-statement')
+class HasStatementFilter(polstmt_filter.HasStatementFilter):
+    def __init__(self, data, manager=None):
+        super().__init__(data, manager)
+        self.policy_attribute = 'AccessPolicies'
+
+    def get_std_format_args(self, domain):
+        return {
+            'domain_arn': domain['ARN'],
+            'account_id': self.manager.config.account_id,
+            'region': self.manager.config.region
+        }
+
+
 @ElasticSearchDomain.action_registry.register('remove-statements')
 class RemovePolicyStatement(RemovePolicyBase):
     """

c7n/resources/rds.py

@@ -986,7 +986,7 @@ class resource_type(TypeInfo):
         enum_spec = ('describe_db_snapshots', 'DBSnapshots', None)
         name = id = 'DBSnapshotIdentifier'
         date = 'SnapshotCreateTime'
-        config_type = "AWS::RDS::DBSnapshot"
+        config_type = cfn_type = "AWS::RDS::DBSnapshot"
         filter_name = "DBSnapshotIdentifier"
         filter_type = "scalar"
         universal_taggable = True

c7n/resources/redshift.py

@@ -804,7 +804,7 @@ class resource_type(TypeInfo):
         enum_spec = ('describe_cluster_snapshots', 'Snapshots', None)
         name = id = 'SnapshotIdentifier'
         date = 'SnapshotCreateTime'
-        config_type = "AWS::Redshift::ClusterSnapshot"
+        config_type = cfn_type = "AWS::Redshift::ClusterSnapshot"
         universal_taggable = True
 
     def get_arns(self, resources):

c7n/resources/sqs.py

@@ -23,7 +23,7 @@
 class DescribeQueue(DescribeSource):
 
     def augment(self, resources):
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
 
         def _augment(r):
             try:
@@ -82,6 +82,29 @@ class resource_type(TypeInfo):
         'config': QueueConfigSource
     }
 
+    def get_client(self):
+        # Work around the fact that boto picks a legacy endpoint by default
+        # which leads to queue urls pointing to legacy instead of standard
+        # which is at odds with config's resource id for the queues.
+        # additionally we need the standard endpoint to work with vpc endpoints.
+        #
+        # sqs canonoical endpoints
+        #  https://docs.aws.amazon.com/general/latest/gr/sqs-service.html
+        # boto3 bug
+        #  https://github.com/boto/botocore/issues/2683 - index of several other bugs
+        #  https://github.com/boto/boto3/issues/1900
+        #
+        # boto3 is transitioning to standard urls per https://github.com/boto/botocore/issues/2705
+        #
+        endpoint = 'https://sqs.{region}.amazonaws.com'.format(region=self.config.region)
+        # these only seem to have the legacy endpoints, so fall through to boto behavior.
+        if self.config.region in ('cn-north-1', 'cn-northwest-1'):
+            endpoint = None
+        params = {}
+        if endpoint:
+            params['endpoint_url'] = endpoint
+        return local_session(self.session_factory).client('sqs', **params)
+
     def get_permissions(self):
         perms = super(SQS, self).get_permissions()
         perms.append('sqs:GetQueueAttributes')
@@ -181,7 +204,7 @@ class RemovePolicyStatement(RemovePolicyBase):
 
     def process(self, resources):
         results = []
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
         for r in resources:
             try:
                 results += filter(None, [self.process_resource(client, r)])
@@ -240,7 +263,7 @@ class ModifyPolicyStatement(ModifyPolicyBase):
 
     def process(self, resources):
         results = []
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
         for r in resources:
             policy = json.loads(r.get('Policy') or '{}')
             policy_statements = policy.setdefault('Statement', [])
@@ -292,7 +315,7 @@ class DeleteSqsQueue(BaseAction):
     permissions = ('sqs:DeleteQueue',)
 
     def process(self, queues):
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
         for q in queues:
             self.process_queue(client, q)
 
@@ -333,7 +356,7 @@ def process(self, queues):
         session = local_session(self.manager.session_factory)
         key_id = session.client(
             'kms').describe_key(KeyId=key)['KeyMetadata']['KeyId']
-        client = session.client('sqs')
+        client = self.manager.get_client()
 
         for q in queues:
             self.process_queue(client, q, key_id)
@@ -376,7 +399,7 @@ class SetRetentionPeriod(BaseAction):
     permissions = ('sqs:SetQueueAttributes',)
 
     def process(self, queues):
-        client = local_session(self.manager.session_factory).client('sqs')
+        client = self.manager.get_client()
         period = str(self.data.get('period', 345600))
         for q in queues:
             client.set_queue_attributes(

c7n/resources/vpc.py

@@ -2057,7 +2057,7 @@ class resource_type(query.TypeInfo):
         id_prefix = 'eipalloc-'
         filter_name = 'AllocationIds'
         filter_type = 'list'
-        config_type = "AWS::EC2::EIP"
+        config_type = cfn_type = "AWS::EC2::EIP"
 
     source_mapping = {
         'describe': DescribeElasticIp,

c7n/version.py

@@ -1,4 +1,2 @@
-# Copyright The Cloud Custodian Authors.
-# SPDX-License-Identifier: Apache-2.0
 # Generated via tools/dev/poetrypkg.py
-version = "0.9.18"
+version = "0.9.19"