Merge branch 'cloud-custodian:master' into develop

Makefile

@@ -61,7 +61,7 @@ pkg-increment:
 	@$(MAKE) pkg-gen-setup
 	python3 tools/dev/poetrypkg.py gen-version-file -p . -f c7n/version.py
 
-pkg-publish-wheel:
+pkg-build-wheel:
 # azure pin uses ancient wheel version, upgrade first
 	pip install -U wheel
 # clean up any artifacts first
@@ -73,6 +73,8 @@ pkg-publish-wheel:
 # check wheel
 	twine check dist/*
 	for pkg in $(PKG_SET); do cd $$pkg && twine check dist/* && cd ../..; done
+
+pkg-publish-wheel:
 # upload to test pypi
 	twine upload -r $(PKG_REPO) dist/*
 	for pkg in $(PKG_SET); do cd $$pkg && twine upload -r $(PKG_REPO) dist/* && cd ../..; done

c7n/resources/cloudfront.py

@@ -155,7 +155,7 @@ class IsWafV2Enabled(Filter):
     def process(self, resources, event=None):
         query = {'Scope': 'CLOUDFRONT'}
         wafs = self.manager.get_resource_manager('wafv2').resources(query, augment=False)
-        waf_name_id_map = {w['Name']: w['Id'] for w in wafs}
+        waf_name_id_map = {w['Name']: w['ARN'] for w in wafs}
         state = self.data.get('state', False)
         target_acl = self.data.get('web-acl')
         target_acl_id = waf_name_id_map.get(target_acl, target_acl)
@@ -343,17 +343,30 @@ class DistributionPostFinding(PostFinding):
 
     def format_resource(self, r):
         envelope, payload = self.format_envelope(r)
-        origins = r['DistributionConfig']['Origins']
+        origins = r['Origins']
 
         payload.update(self.filter_empty({
             'DomainName': r['DomainName'],
             'WebACLId': r.get('WebACLId'),
-            'LastModifiedTime': r['LastModifiedTime'],
+            'LastModifiedTime': r['LastModifiedTime'].isoformat(),
             'Status': r['Status'],
-            'Logging': self.filter_empty(r['DistributionConfig'].get('Logging', {})),
-            'Origins': [
-                dict(Id=o['Id'], OriginPath=o['OriginPath'], DomainName=o['DomainName'])
-                for o in origins]
+            'Logging': self.filter_empty(r.get('Logging', {})),
+            'Origins': {
+                'Items': [
+                    {
+                        # Extract a subset of origin item keys,
+                        # only if they're non-empty.
+                        #
+                        # The full item can be large and noisy, and
+                        # empty string values (notably for OriginPath)
+                        # will fail validation.
+                        k: o[k]
+                        for k in self.filter_empty(o)
+                        if k in ('Id', 'OriginPath', 'DomainName')
+                    }
+                    for o in origins['Items']
+                ]
+            }
         }))
 
         return envelope
@@ -410,7 +423,7 @@ class SetWafv2(BaseAction):
     def process(self, resources):
         query = {'Scope': 'CLOUDFRONT'}
         wafs = self.manager.get_resource_manager('wafv2').resources(query, augment=False)
-        waf_name_id_map = {w['Name']: w['Id'] for w in wafs}
+        waf_name_id_map = {w['Name']: w['ARN'] for w in wafs}
         target_acl = self.data.get('web-acl')
         target_acl_id = waf_name_id_map.get(target_acl, target_acl)
         if target_acl_id not in waf_name_id_map.values():

c7n/resources/cloudsearch.py

@@ -2,13 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 from c7n.actions import Action
 from c7n.manager import resources
+from c7n.filters import ValueFilter
 from c7n.query import QueryResourceManager, TypeInfo
 from c7n.utils import local_session, type_schema
 
 
 @resources.register('cloudsearch')
 class CloudSearch(QueryResourceManager):
-
     class resource_type(TypeInfo):
         service = "cloudsearch"
         enum_spec = ("describe_domains", "DomainStatusList", None)
@@ -21,7 +21,6 @@ class resource_type(TypeInfo):
 
 @CloudSearch.action_registry.register('delete')
 class Delete(Action):
-
     schema = type_schema('delete')
     permissions = ('cloudsearch:DeleteDomain',)
 
@@ -32,3 +31,75 @@ def process(self, resources):
             if r['Created'] is not True or r['Deleted'] is True:
                 continue
             client.delete_domain(DomainName=r['DomainName'])
+
+
+@CloudSearch.filter_registry.register('domain-options')
+class DomainOptionsFilter(ValueFilter):
+    """
+    Filter for cloud search domains by their domain options.
+
+    :example:
+
+    .. code-block:: yaml
+
+            policies:
+              - name: cloudsearch-detect-https
+                resource: cloudsearch
+                filters:
+                  - type: domain-options
+                    key: Options.EnforceHTTPS
+                    value: false
+
+    """
+
+    schema = type_schema('domain-options', rinherit=ValueFilter.schema)
+    permissions = ('cloudsearch:DescribeDomainEndpointOptions',)
+
+    def process(self, resources, event=None):
+        results = []
+        client = local_session(self.manager.session_factory).client('cloudsearch')
+        for r in resources:
+            options = client.describe_domain_endpoint_options(
+                DomainName=r['DomainName']
+            ).get('DomainEndpointOptions')
+            if self.match(options):
+                results.append(r)
+        return results
+
+
+@CloudSearch.action_registry.register('enable-https')
+class EnableHttps(Action):
+    """Enable HTTPs to cloudsearch
+
+    :example:
+    .. code-block:: yaml
+            policies:
+              - name: enable-https
+                resource: cloudsearch
+                filters:
+                  - type: domain-options
+                    key: Options.EnforceHTTPS
+                    value: false
+                actions:
+                  - type: enable-https
+                    tls-policy: Policy-Min-TLS-1-0-2019-07
+    """
+
+    schema = type_schema(
+        'enable-https',
+        **{"tls-policy": {'enum': ['Policy-Min-TLS-1-0-2019-07', 'Policy-Min-TLS-1-2-2019-07']}}
+    )
+    permissions = ('cloudsearch:UpdateDomainEndpointOptions',)
+
+    def process(self, resources):
+        client = local_session(
+            self.manager.session_factory).client('cloudsearch')
+        for r in resources:
+            client.update_domain_endpoint_options(
+                DomainName=r['DomainName'],
+                DomainEndpointOptions={
+                    'EnforceHTTPS': True,
+                    'TLSSecurityPolicy': self.data.get(
+                        'tls-policy', 'Policy-Min-TLS-1-2-2019-07')
+                }
+            )

c7n/resources/iam.py

@@ -1081,13 +1081,33 @@ class SetPolicy(BaseAction):
         arn={'type': 'string'},
         required=['state', 'arn'])
 
-    permissions = ('iam:AttachRolePolicy', 'iam:DetachRolePolicy',)
+    permissions = ('iam:AttachRolePolicy', 'iam:DetachRolePolicy', "iam:ListAttachedRolePolicies",)
 
     def validate(self):
         if self.data.get('state') == 'attached' and self.data.get('arn') == "*":
             raise PolicyValidationError(
                 '* operator is not supported for state: attached on %s' % (self.manager.data))
 
+    def attach_policy(self, client, resource, policy_arn):
+        client.attach_role_policy(
+            RoleName=resource['RoleName'],
+            PolicyArn=policy_arn
+        )
+
+    def detach_policy(self, client, resource, policy_arn):
+        try:
+            client.detach_role_policy(
+                RoleName=resource['RoleName'],
+                PolicyArn=policy_arn
+            )
+        except client.exceptions.NoSuchEntityException:
+            return
+
+    def list_attached_policies(self, client, resource):
+        attached_policy = client.list_attached_role_policies(RoleName=resource['RoleName'])
+        policy_arns = [p.get('PolicyArn') for p in attached_policy['AttachedPolicies']]
+        return policy_arns
+
     def process(self, resources):
         client = local_session(self.manager.session_factory).client('iam')
         policy_arn = self.data['arn']
@@ -1098,27 +1118,68 @@ def process(self, resources):
         state = self.data['state']
         for r in resources:
             if state == 'attached':
-                client.attach_role_policy(
-                    RoleName=r['RoleName'],
-                    PolicyArn=policy_arn)
+                self.attach_policy(client, r, policy_arn)
             elif state == 'detached' and policy_arn != "*":
-                try:
-                    client.detach_role_policy(
-                        RoleName=r['RoleName'],
-                        PolicyArn=policy_arn)
-                except client.exceptions.NoSuchEntityException:
-                    continue
+                self.detach_policy(client, r, policy_arn)
             elif state == 'detached' and policy_arn == "*":
                 try:
                     self.detach_all_policies(client, r)
                 except client.exceptions.NoSuchEntityException:
                     continue
 
     def detach_all_policies(self, client, resource):
-        attached_policy = client.list_attached_role_policies(RoleName=resource['RoleName'])
-        policy_arns = [p.get('PolicyArn') for p in attached_policy['AttachedPolicies']]
+        policy_arns = self.list_attached_policies(client, resource)
         for parn in policy_arns:
-            client.detach_role_policy(RoleName=resource['RoleName'], PolicyArn=parn)
+            self.detach_policy(client, resource, parn)
+
+
+@Group.action_registry.register("set-policy")
+class SetGroupPolicy(SetPolicy):
+    """Set a specific IAM policy as attached or detached on a group.
+
+    You will identify the policy by its arn.
+
+    Returns a list of roles modified by the action.
+
+    For example, if you want to automatically attach a single policy while
+    detaching all exisitng policies:
+
+    :example:
+
+      .. code-block:: yaml
+
+        - name: iam-attach-group-policy
+          resource: iam-group
+          actions:
+            - type: set-policy
+              state: detached
+              arn: "*"
+            - type: set-policy
+              state: attached
+              arn: arn:aws:iam::{account_id}:policy/my-iam-policy
+
+    """
+
+    permissions = (
+        "iam:AttachGroupPolicy", "iam:DetachGroupPolicy", "iam:ListAttachedGroupPolicies",)
+
+    def attach_policy(self, client, resource, policy_arn):
+        client.attach_group_policy(
+            GroupName=resource["GroupName"], PolicyArn=policy_arn)
+
+    def detach_policy(self, client, resource, policy_arn):
+        try:
+            client.detach_group_policy(
+                GroupName=resource["GroupName"], PolicyArn=policy_arn)
+        except client.exceptions.NoSuchEntityException:
+            return
+
+    def list_attached_policies(self, client, resource):
+        attached_policies = client.list_attached_group_policies(
+            GroupName=resource["GroupName"]
+        )
+        policy_arns = [p.get('PolicyArn') for p in attached_policies['AttachedPolicies']]
+        return policy_arns
 
 
 @Role.action_registry.register('delete')

c7n/resources/ssm.py

@@ -795,9 +795,8 @@ class resource_type(TypeInfo):
 
         enum_spec = ('list_resource_data_sync', 'ResourceDataSyncItems', None)
         service = 'ssm'
-        arn_type = 'datasync'
-        id = 'DataSync'
-        name = 'Title'
+        arn_type = 'resource-data-sync'
+        id = name = 'SyncName'
 
     permissions = ('ssm:ListResourceDataSync',)
 

c7n/tags.py

@@ -880,8 +880,11 @@ def process_resource_set(self, client, resource_set, tags):
             client.tag_resources, ResourceARNList=arns, Tags=tags)
 
     def get_client(self):
-        return utils.local_session(
-            self.manager.session_factory).client('resourcegroupstaggingapi')
+        # For global resources, manage tags from us-east-1
+        region = (getattr(self.manager.resource_type, 'global_resource', None)
+            and 'us-east-1' or self.manager.region)
+        return utils.local_session(self.manager.session_factory).client(
+            'resourcegroupstaggingapi', region_name=region)
 
 
 class UniversalUntag(RemoveTag):
@@ -893,8 +896,11 @@ class UniversalUntag(RemoveTag):
     permissions = ('tag:UntagResources',)
 
     def get_client(self):
-        return utils.local_session(
-            self.manager.session_factory).client('resourcegroupstaggingapi')
+        # For global resources, manage tags from us-east-1
+        region = (getattr(self.manager.resource_type, 'global_resource', None)
+            and 'us-east-1' or self.manager.region)
+        return utils.local_session(self.manager.session_factory).client(
+            'resourcegroupstaggingapi', region_name=region)
 
     def process_resource_set(self, client, resource_set, tag_keys):
         arns = self.manager.get_arns(resource_set)
@@ -964,8 +970,11 @@ def process_resource_set(self, client, resource_set, tags):
             client.tag_resources, ResourceARNList=arns, Tags=tags)
 
     def get_client(self):
-        return utils.local_session(
-            self.manager.session_factory).client('resourcegroupstaggingapi')
+        # For global resources, manage tags from us-east-1
+        region = (getattr(self.manager.resource_type, 'global_resource', None)
+            and 'us-east-1' or self.manager.region)
+        return utils.local_session(self.manager.session_factory).client(
+            'resourcegroupstaggingapi', region_name=region)
 
 
 class CopyRelatedResourceTag(Tag):

c7n/version.py

@@ -1,2 +1,2 @@
 # Generated via tools/dev/poetrypkg.py
-version = "0.9.16"
+version = "0.9.17"