From 22e8de53aee7b324f9e0865238ec933cc9fa1eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anto=C5=9B=20Bu=C4=87ko?= <30908515+Paladin-Dranser@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:57:06 +0200 Subject: [PATCH 1/3] aws - ec2 - stop-protected filter for disableApiStop attribute (#7608) --- c7n/commands.py | 5 +- c7n/policy.py | 11 +- c7n/query.py | 3 +- c7n/resources/ec2.py | 59 +++ .../ec2.DescribeInstanceAttribute_1.json | 10 + .../ec2.DescribeInstanceAttribute_2.json | 10 + .../ec2.DescribeInstanceAttribute_3.json | 10 + .../ec2.DescribeInstances_1.json | 494 ++++++++++++++++++ .../ec2.DescribeTags_1.json | 7 + .../ec2.DescribeInstanceAttribute_1.json | 10 + .../ec2.DescribeInstanceAttribute_2.json | 10 + .../ec2.DescribeInstanceAttribute_3.json | 10 + .../ec2.DescribeInstances_1.json | 494 ++++++++++++++++++ .../ec2.DescribeTags_1.json | 7 + .../ec2_stop_protection_disabled/ami.tf | 12 + .../ec2_stop_protection_disabled/main.tf | 19 + .../ec2_stop_protection_disabled/network.tf | 8 + .../tf_resources.json | 434 +++++++++++++++ .../ec2_stop_protection_enabled/ami.tf | 12 + .../ec2_stop_protection_enabled/main.tf | 19 + .../ec2_stop_protection_enabled/network.tf | 8 + .../tf_resources.json | 434 +++++++++++++++ tests/test_ec2.py | 111 ++++ 23 files changed, 2189 insertions(+), 8 deletions(-) create mode 100644 tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_1.json create mode 100644 tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_2.json create mode 100644 tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_3.json create mode 100644 tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstances_1.json create mode 100644 tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeTags_1.json create mode 100644 tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_1.json create mode 100644 tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_2.json create mode 100644 tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_3.json create mode 100644 tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstances_1.json create mode 100644 tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeTags_1.json create mode 100644 tests/terraform/ec2_stop_protection_disabled/ami.tf create mode 100644 tests/terraform/ec2_stop_protection_disabled/main.tf create mode 100644 tests/terraform/ec2_stop_protection_disabled/network.tf create mode 100644 tests/terraform/ec2_stop_protection_disabled/tf_resources.json create mode 100644 tests/terraform/ec2_stop_protection_enabled/ami.tf create mode 100644 tests/terraform/ec2_stop_protection_enabled/main.tf create mode 100644 tests/terraform/ec2_stop_protection_enabled/network.tf create mode 100644 tests/terraform/ec2_stop_protection_enabled/tf_resources.json diff --git a/c7n/commands.py b/c7n/commands.py index fc83e06fa45..16eed1a2c4d 100644 --- a/c7n/commands.py +++ b/c7n/commands.py @@ -8,6 +8,7 @@ import logging import os import sys +from typing import List import yaml from yaml.constructor import ConstructorError @@ -283,7 +284,7 @@ def validate(options): @policy_command -def run(options, policies): +def run(options, policies: List[Policy]) -> None: exit_code = 0 # AWS - Sanity check that we have an assumable role before executing policies @@ -295,7 +296,7 @@ def run(options, policies): log.exception("Unable to assume role %s", options.assume_role) sys.exit(1) - errored_policies = [] + errored_policies: List[str] = [] for policy in policies: try: policy() diff --git a/c7n/policy.py b/c7n/policy.py index 3336acbe23b..3093925c029 100644 --- a/c7n/policy.py +++ b/c7n/policy.py @@ -7,6 +7,7 @@ import logging import os import time +from typing import List from dateutil import parser, tz as tzutil import jmespath @@ -66,12 +67,12 @@ class PolicyCollection: log = logging.getLogger('c7n.policies') - def __init__(self, policies, options): + def __init__(self, policies: 'List[Policy]', options): self.options = options self.policies = policies @classmethod - def from_data(cls, data, options, session_factory=None): + def from_data(cls, data: dict, options, session_factory=None): # session factory param introduction needs an audit and review # on tests. sf = session_factory if session_factory else cls.session_factory() @@ -1100,15 +1101,15 @@ def __repr__(self): self.resource_type, self.name, self.options.region) @property - def name(self): + def name(self) -> str: return self.data['name'] @property - def resource_type(self): + def resource_type(self) -> str: return self.data['resource'] @property - def provider_name(self): + def provider_name(self) -> str: if '.' in self.resource_type: provider_name, resource_type = self.resource_type.split('.', 1) else: diff --git a/c7n/query.py b/c7n/query.py index 907920ae279..d16e25c3875 100644 --- a/c7n/query.py +++ b/c7n/query.py @@ -9,6 +9,7 @@ import functools import itertools import json +from typing import List import jmespath import os @@ -503,7 +504,7 @@ def get_cache_key(self, query): 'q': query } - def resources(self, query=None, augment=True): + def resources(self, query=None, augment=True) -> List[dict]: query = self.source.get_query_params(query) cache_key = self.get_cache_key(query) resources = None diff --git a/c7n/resources/ec2.py b/c7n/resources/ec2.py index 8ca87f54b8e..bdf6a2cfa22 100644 --- a/c7n/resources/ec2.py +++ b/c7n/resources/ec2.py @@ -6,7 +6,10 @@ import random import re import zlib +from typing import List +from distutils.version import LooseVersion +import botocore from botocore.exceptions import ClientError from dateutil.parser import parse from concurrent.futures import as_completed @@ -295,6 +298,62 @@ def __call__(self, i): return self.operator(map(self.match, volumes)) +@filters.register('stop-protected') +class DisableApiStop(Filter): + """EC2 instances with ``disableApiStop`` attribute set + + Filters EC2 instances with ``disableApiStop`` attribute set to true. + + :Example: + + .. code-block:: yaml + + policies: + - name: stop-protection-enabled + resource: ec2 + filters: + - type: stop-protected + + :Example: + + .. code-block:: yaml + + policies: + - name: stop-protection-NOT-enabled + resource: ec2 + filters: + - not: + - type: stop-protected + """ + + schema = type_schema('stop-protected') + permissions = ('ec2:DescribeInstanceAttribute',) + + def process(self, resources: List[dict], event=None) -> List[dict]: + client = utils.local_session( + self.manager.session_factory).client('ec2') + return [r for r in resources + if self._is_stop_protection_enabled(client, r)] + + def _is_stop_protection_enabled(self, client, instance: dict) -> bool: + attr_val = self.manager.retry( + client.describe_instance_attribute, + Attribute='disableApiStop', + InstanceId=instance['InstanceId'] + ) + return attr_val['DisableApiStop']['Value'] + + def validate(self) -> None: + botocore_min_version = '1.26.7' + + if LooseVersion(botocore.__version__) < LooseVersion(botocore_min_version): + raise PolicyValidationError( + "'stop-protected' filter requires botocore version " + f'{botocore_min_version} or above. ' + f'Installed version is {botocore.__version__}.' + ) + + @filters.register('termination-protected') class DisableApiTermination(Filter): """EC2 instances with ``disableApiTermination`` attribute set diff --git a/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_1.json b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_1.json new file mode 100644 index 00000000000..a6baf6ac396 --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_1.json @@ -0,0 +1,10 @@ +{ + "status_code": 200, + "data": { + "InstanceId": "i-0ace8b3999421ba93", + "DisableApiStop": { + "Value": false + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_2.json b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_2.json new file mode 100644 index 00000000000..cfc93df8fa2 --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_2.json @@ -0,0 +1,10 @@ +{ + "status_code": 200, + "data": { + "InstanceId": "i-0916290676d3ede6a", + "DisableApiStop": { + "Value": false + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_3.json b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_3.json new file mode 100644 index 00000000000..a05b1be4cf3 --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstanceAttribute_3.json @@ -0,0 +1,10 @@ +{ + "status_code": 200, + "data": { + "InstanceId": "i-00737300785a058f9", + "DisableApiStop": { + "Value": true + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstances_1.json b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstances_1.json new file mode 100644 index 00000000000..e5afb649dc9 --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeInstances_1.json @@ -0,0 +1,494 @@ +{ + "status_code": 200, + "data": { + "Reservations": [ + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0d7734f53f491186b", + "InstanceId": "i-0ace8b3999421ba93", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 10, + "microsecond": 0 + }, + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-east-1b", + "GroupName": "", + "Tenancy": "default" + }, + "PrivateDnsName": "ip-10-0-1-8.ec2.internal", + "PrivateIpAddress": "10.0.1.8", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 16, + "Name": "running" + }, + "StateTransitionReason": "", + "SubnetId": "subnet-0097c16e28e99cd1f", + "VpcId": "vpc-0bf64b04425b5bd8f", + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 11, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-069a6298487605445" + } + } + ], + "ClientToken": "CEAC0FD7-9EB9-42E1-AB7B-46DB7347B928", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 10, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-0006e2995c35f7dfa", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "default", + "GroupId": "sg-0b19c3cb919a3fe08" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:2e:58:2f:2c:43", + "NetworkInterfaceId": "eni-038d5123afd6dee32", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.8", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.8" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-0097c16e28e99cd1f", + "VpcId": "vpc-0bf64b04425b5bd8f", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "default", + "GroupId": "sg-0b19c3cb919a3fe08" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 10, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + } + } + ], + "OwnerId": "644160558196", + "ReservationId": "r-01611ca4e7520b17c" + }, + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0d7734f53f491186b", + "InstanceId": "i-0916290676d3ede6a", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 10, + "microsecond": 0 + }, + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-east-1b", + "GroupName": "", + "Tenancy": "default" + }, + "PrivateDnsName": "ip-10-0-1-227.ec2.internal", + "PrivateIpAddress": "10.0.1.227", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 16, + "Name": "running" + }, + "StateTransitionReason": "", + "SubnetId": "subnet-0097c16e28e99cd1f", + "VpcId": "vpc-0bf64b04425b5bd8f", + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 11, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-0f19e8a86522e29fa" + } + } + ], + "ClientToken": "460B0613-4782-4EA8-8B1C-2C60C1A219B1", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 10, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-0e4110e67f1585b8b", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "default", + "GroupId": "sg-0b19c3cb919a3fe08" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:87:d4:cd:9c:ff", + "NetworkInterfaceId": "eni-0cf997427424304d4", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.227", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.227" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-0097c16e28e99cd1f", + "VpcId": "vpc-0bf64b04425b5bd8f", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "default", + "GroupId": "sg-0b19c3cb919a3fe08" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 10, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + } + } + ], + "OwnerId": "644160558196", + "ReservationId": "r-08b33f8edee2c1956" + }, + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0d7734f53f491186b", + "InstanceId": "i-00737300785a058f9", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 11, + "microsecond": 0 + }, + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-east-1b", + "GroupName": "", + "Tenancy": "default" + }, + "PrivateDnsName": "ip-10-0-1-224.ec2.internal", + "PrivateIpAddress": "10.0.1.224", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 16, + "Name": "running" + }, + "StateTransitionReason": "", + "SubnetId": "subnet-0097c16e28e99cd1f", + "VpcId": "vpc-0bf64b04425b5bd8f", + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 11, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-0911be7318df80a51" + } + } + ], + "ClientToken": "8E7C2B80-05F1-493B-8935-C3C57506015C", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 11, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-01cd8b8d5574ea995", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "default", + "GroupId": "sg-0b19c3cb919a3fe08" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:55:4b:96:45:c5", + "NetworkInterfaceId": "eni-09bf69ce93aa23e6d", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.224", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.224" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-0097c16e28e99cd1f", + "VpcId": "vpc-0bf64b04425b5bd8f", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "default", + "GroupId": "sg-0b19c3cb919a3fe08" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 51, + "second": 10, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + } + } + ], + "OwnerId": "644160558196", + "ReservationId": "r-09a31d594eee1cb78" + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeTags_1.json b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeTags_1.json new file mode 100644 index 00000000000..e41b16b3e98 --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_disabled/ec2.DescribeTags_1.json @@ -0,0 +1,7 @@ +{ + "status_code": 200, + "data": { + "Tags": [], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_1.json b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_1.json new file mode 100644 index 00000000000..83b048eacfe --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_1.json @@ -0,0 +1,10 @@ +{ + "status_code": 200, + "data": { + "InstanceId": "i-017e55c722d539f90", + "DisableApiStop": { + "Value": false + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_2.json b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_2.json new file mode 100644 index 00000000000..4b2d162dbeb --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_2.json @@ -0,0 +1,10 @@ +{ + "status_code": 200, + "data": { + "InstanceId": "i-0daf36c03bf9c5e75", + "DisableApiStop": { + "Value": false + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_3.json b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_3.json new file mode 100644 index 00000000000..7100bbd12cd --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstanceAttribute_3.json @@ -0,0 +1,10 @@ +{ + "status_code": 200, + "data": { + "InstanceId": "i-0ecfe7c50cf2f76b2", + "DisableApiStop": { + "Value": true + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstances_1.json b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstances_1.json new file mode 100644 index 00000000000..48767de443d --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeInstances_1.json @@ -0,0 +1,494 @@ +{ + "status_code": 200, + "data": { + "Reservations": [ + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0d7734f53f491186b", + "InstanceId": "i-017e55c722d539f90", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-east-1f", + "GroupName": "", + "Tenancy": "default" + }, + "PrivateDnsName": "ip-10-0-1-168.ec2.internal", + "PrivateIpAddress": "10.0.1.168", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 16, + "Name": "running" + }, + "StateTransitionReason": "", + "SubnetId": "subnet-0e172a55c20169303", + "VpcId": "vpc-00255c1f52c0a88d6", + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-0f1ee2360fa73f764" + } + } + ], + "ClientToken": "78C6248B-3FFA-4B29-874F-0AE73674F2C4", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-0fded6ac2e059ed5d", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "default", + "GroupId": "sg-09c6a03dec497a6b0" + } + ], + "Ipv6Addresses": [], + "MacAddress": "16:b2:be:00:93:87", + "NetworkInterfaceId": "eni-004e46d428e6b695a", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.168", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.168" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-0e172a55c20169303", + "VpcId": "vpc-00255c1f52c0a88d6", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "default", + "GroupId": "sg-09c6a03dec497a6b0" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + } + } + ], + "OwnerId": "644160558196", + "ReservationId": "r-0cb1509e933be4992" + }, + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0d7734f53f491186b", + "InstanceId": "i-0daf36c03bf9c5e75", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-east-1f", + "GroupName": "", + "Tenancy": "default" + }, + "PrivateDnsName": "ip-10-0-1-15.ec2.internal", + "PrivateIpAddress": "10.0.1.15", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 16, + "Name": "running" + }, + "StateTransitionReason": "", + "SubnetId": "subnet-0e172a55c20169303", + "VpcId": "vpc-00255c1f52c0a88d6", + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-07fccc8f6c75d1d1c" + } + } + ], + "ClientToken": "5671183C-7176-4D0F-93C7-E0703AA88F2B", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-005f4b0f31ceb16b1", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "default", + "GroupId": "sg-09c6a03dec497a6b0" + } + ], + "Ipv6Addresses": [], + "MacAddress": "16:2f:ca:48:69:fd", + "NetworkInterfaceId": "eni-0d3a0342e5b7b9403", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.15", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.15" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-0e172a55c20169303", + "VpcId": "vpc-00255c1f52c0a88d6", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "default", + "GroupId": "sg-09c6a03dec497a6b0" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + } + } + ], + "OwnerId": "644160558196", + "ReservationId": "r-076e7b153ee60b37a" + }, + { + "Groups": [], + "Instances": [ + { + "AmiLaunchIndex": 0, + "ImageId": "ami-0d7734f53f491186b", + "InstanceId": "i-0ecfe7c50cf2f76b2", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "Monitoring": { + "State": "disabled" + }, + "Placement": { + "AvailabilityZone": "us-east-1f", + "GroupName": "", + "Tenancy": "default" + }, + "PrivateDnsName": "ip-10-0-1-124.ec2.internal", + "PrivateIpAddress": "10.0.1.124", + "ProductCodes": [], + "PublicDnsName": "", + "State": { + "Code": 16, + "Name": "running" + }, + "StateTransitionReason": "", + "SubnetId": "subnet-0e172a55c20169303", + "VpcId": "vpc-00255c1f52c0a88d6", + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-0d2b762057cf67207" + } + } + ], + "ClientToken": "1EB6F15E-842A-4A68-914B-818224E485B4", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-025d5e6d9b9d22787", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupName": "default", + "GroupId": "sg-09c6a03dec497a6b0" + } + ], + "Ipv6Addresses": [], + "MacAddress": "16:46:43:6b:2a:4d", + "NetworkInterfaceId": "eni-0d6d13c72ac812cae", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.124", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.124" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-0e172a55c20169303", + "VpcId": "vpc-00255c1f52c0a88d6", + "InterfaceType": "interface" + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupName": "default", + "GroupId": "sg-09c6a03dec497a6b0" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2022, + "month": 8, + "day": 4, + "hour": 11, + "minute": 33, + "second": 49, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default" + } + } + ], + "OwnerId": "644160558196", + "ReservationId": "r-0306be6c4abf309e0" + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeTags_1.json b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeTags_1.json new file mode 100644 index 00000000000..e41b16b3e98 --- /dev/null +++ b/tests/data/placebo/ec2_stop_protection_enabled/ec2.DescribeTags_1.json @@ -0,0 +1,7 @@ +{ + "status_code": 200, + "data": { + "Tags": [], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/terraform/ec2_stop_protection_disabled/ami.tf b/tests/terraform/ec2_stop_protection_disabled/ami.tf new file mode 100644 index 00000000000..754055ff1c4 --- /dev/null +++ b/tests/terraform/ec2_stop_protection_disabled/ami.tf @@ -0,0 +1,12 @@ +data "aws_ami" "amazon_linux" { + most_recent = true + + owners = ["amazon"] + + filter { + name = "name" + values = [ + "amzn-ami-hvm-*-x86_64-gp2", + ] + } +} diff --git a/tests/terraform/ec2_stop_protection_disabled/main.tf b/tests/terraform/ec2_stop_protection_disabled/main.tf new file mode 100644 index 00000000000..f63b7536502 --- /dev/null +++ b/tests/terraform/ec2_stop_protection_disabled/main.tf @@ -0,0 +1,19 @@ +resource "aws_instance" "no_protection" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id +} + +resource "aws_instance" "termination_protection" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id + disable_api_termination = true +} + +resource "aws_instance" "stop_protection" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id + disable_api_stop = true +} diff --git a/tests/terraform/ec2_stop_protection_disabled/network.tf b/tests/terraform/ec2_stop_protection_disabled/network.tf new file mode 100644 index 00000000000..3c95d7488b0 --- /dev/null +++ b/tests/terraform/ec2_stop_protection_disabled/network.tf @@ -0,0 +1,8 @@ +resource "aws_vpc" "example" { + cidr_block = "10.0.0.0/16" +} + +resource "aws_subnet" "example" { + vpc_id = aws_vpc.example.id + cidr_block = "10.0.1.0/24" +} diff --git a/tests/terraform/ec2_stop_protection_disabled/tf_resources.json b/tests/terraform/ec2_stop_protection_disabled/tf_resources.json new file mode 100644 index 00000000000..475f0b3cd81 --- /dev/null +++ b/tests/terraform/ec2_stop_protection_disabled/tf_resources.json @@ -0,0 +1,434 @@ +{ + "pytest-terraform": 1, + "outputs": {}, + "resources": { + "aws_ami": { + "amazon_linux": { + "architecture": "x86_64", + "arn": "arn:aws:ec2:us-east-1::image/ami-0d7734f53f491186b", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": { + "delete_on_termination": "true", + "encrypted": "false", + "iops": "0", + "snapshot_id": "snap-00e9c002a992550a7", + "throughput": "0", + "volume_size": "8", + "volume_type": "gp2" + }, + "no_device": "", + "virtual_name": "" + } + ], + "boot_mode": "", + "creation_date": "2022-07-16T02:39:01.000Z", + "deprecation_time": "2024-07-16T02:39:01.000Z", + "description": "Amazon Linux AMI 2018.03.0.20220705.1 x86_64 HVM gp2", + "ena_support": true, + "executable_users": null, + "filter": [ + { + "name": "name", + "values": [ + "amzn-ami-hvm-*-x86_64-gp2" + ] + } + ], + "hypervisor": "xen", + "id": "ami-0d7734f53f491186b", + "image_id": "ami-0d7734f53f491186b", + "image_location": "amazon/amzn-ami-hvm-2018.03.0.20220705.1-x86_64-gp2", + "image_owner_alias": "amazon", + "image_type": "machine", + "include_deprecated": false, + "kernel_id": "", + "most_recent": true, + "name": "amzn-ami-hvm-2018.03.0.20220705.1-x86_64-gp2", + "name_regex": null, + "owner_id": "644160558196", + "owners": [ + "amazon" + ], + "platform": "", + "platform_details": "Linux/UNIX", + "product_codes": [], + "public": true, + "ramdisk_id": "", + "root_device_name": "/dev/xvda", + "root_device_type": "ebs", + "root_snapshot_id": "snap-00e9c002a992550a7", + "sriov_net_support": "simple", + "state": "available", + "state_reason": { + "code": "UNSET", + "message": "UNSET" + }, + "tags": {}, + "tpm_support": "", + "usage_operation": "RunInstances", + "virtualization_type": "hvm" + } + }, + "aws_instance": { + "no_protection": { + "ami": "ami-0d7734f53f491186b", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-0916290676d3ede6a", + "associate_public_ip_address": false, + "availability_zone": "us-east-1b", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 1, + "cpu_threads_per_core": 1, + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": false, + "disable_api_termination": false, + "ebs_block_device": [], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "", + "id": "i-0916290676d3ede6a", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-0cf997427424304d4", + "private_dns": "ip-10-0-1-227.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.227", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": true, + "iops": 100, + "kms_key_id": "arn:aws:kms:us-east-1:644160558196:key/9ee2a815-01df-4571-9946-3464baabfacd", + "tags": {}, + "throughput": 0, + "volume_id": "vol-0f19e8a86522e29fa", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-0097c16e28e99cd1f", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-0b19c3cb919a3fe08" + ] + }, + "stop_protection": { + "ami": "ami-0d7734f53f491186b", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-00737300785a058f9", + "associate_public_ip_address": false, + "availability_zone": "us-east-1b", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 1, + "cpu_threads_per_core": 1, + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": true, + "disable_api_termination": false, + "ebs_block_device": [], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "", + "id": "i-00737300785a058f9", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-09bf69ce93aa23e6d", + "private_dns": "ip-10-0-1-224.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.224", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": true, + "iops": 100, + "kms_key_id": "arn:aws:kms:us-east-1:644160558196:key/9ee2a815-01df-4571-9946-3464baabfacd", + "tags": {}, + "throughput": 0, + "volume_id": "vol-0911be7318df80a51", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-0097c16e28e99cd1f", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-0b19c3cb919a3fe08" + ] + }, + "termination_protection": { + "ami": "ami-0d7734f53f491186b", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-0ace8b3999421ba93", + "associate_public_ip_address": false, + "availability_zone": "us-east-1b", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 1, + "cpu_threads_per_core": 1, + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": false, + "disable_api_termination": true, + "ebs_block_device": [], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "", + "id": "i-0ace8b3999421ba93", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-038d5123afd6dee32", + "private_dns": "ip-10-0-1-8.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.8", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": true, + "iops": 100, + "kms_key_id": "arn:aws:kms:us-east-1:644160558196:key/9ee2a815-01df-4571-9946-3464baabfacd", + "tags": {}, + "throughput": 0, + "volume_id": "vol-069a6298487605445", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-0097c16e28e99cd1f", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-0b19c3cb919a3fe08" + ] + } + }, + "aws_subnet": { + "example": { + "arn": "arn:aws:ec2:us-east-1:644160558196:subnet/subnet-0097c16e28e99cd1f", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-1b", + "availability_zone_id": "use1-az2", + "cidr_block": "10.0.1.0/24", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-0097c16e28e99cd1f", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_native": false, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "644160558196", + "private_dns_hostname_type_on_launch": "ip-name", + "tags": null, + "tags_all": {}, + "timeouts": null, + "vpc_id": "vpc-0bf64b04425b5bd8f" + } + }, + "aws_vpc": { + "example": { + "arn": "arn:aws:ec2:us-east-1:644160558196:vpc/vpc-0bf64b04425b5bd8f", + "assign_generated_ipv6_cidr_block": false, + "cidr_block": "10.0.0.0/16", + "default_network_acl_id": "acl-061af6d2449d2183f", + "default_route_table_id": "rtb-0b957138d7879cd8b", + "default_security_group_id": "sg-0b19c3cb919a3fe08", + "dhcp_options_id": "dopt-041fd1396ba62bfa4", + "enable_classiclink": false, + "enable_classiclink_dns_support": false, + "enable_dns_hostnames": false, + "enable_dns_support": true, + "id": "vpc-0bf64b04425b5bd8f", + "instance_tenancy": "default", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_association_id": "", + "ipv6_cidr_block": "", + "ipv6_cidr_block_network_border_group": "", + "ipv6_ipam_pool_id": "", + "ipv6_netmask_length": 0, + "main_route_table_id": "rtb-0b957138d7879cd8b", + "owner_id": "644160558196", + "tags": null, + "tags_all": {} + } + } + } +} \ No newline at end of file diff --git a/tests/terraform/ec2_stop_protection_enabled/ami.tf b/tests/terraform/ec2_stop_protection_enabled/ami.tf new file mode 100644 index 00000000000..754055ff1c4 --- /dev/null +++ b/tests/terraform/ec2_stop_protection_enabled/ami.tf @@ -0,0 +1,12 @@ +data "aws_ami" "amazon_linux" { + most_recent = true + + owners = ["amazon"] + + filter { + name = "name" + values = [ + "amzn-ami-hvm-*-x86_64-gp2", + ] + } +} diff --git a/tests/terraform/ec2_stop_protection_enabled/main.tf b/tests/terraform/ec2_stop_protection_enabled/main.tf new file mode 100644 index 00000000000..f63b7536502 --- /dev/null +++ b/tests/terraform/ec2_stop_protection_enabled/main.tf @@ -0,0 +1,19 @@ +resource "aws_instance" "no_protection" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id +} + +resource "aws_instance" "termination_protection" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id + disable_api_termination = true +} + +resource "aws_instance" "stop_protection" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id + disable_api_stop = true +} diff --git a/tests/terraform/ec2_stop_protection_enabled/network.tf b/tests/terraform/ec2_stop_protection_enabled/network.tf new file mode 100644 index 00000000000..3c95d7488b0 --- /dev/null +++ b/tests/terraform/ec2_stop_protection_enabled/network.tf @@ -0,0 +1,8 @@ +resource "aws_vpc" "example" { + cidr_block = "10.0.0.0/16" +} + +resource "aws_subnet" "example" { + vpc_id = aws_vpc.example.id + cidr_block = "10.0.1.0/24" +} diff --git a/tests/terraform/ec2_stop_protection_enabled/tf_resources.json b/tests/terraform/ec2_stop_protection_enabled/tf_resources.json new file mode 100644 index 00000000000..edf39639af7 --- /dev/null +++ b/tests/terraform/ec2_stop_protection_enabled/tf_resources.json @@ -0,0 +1,434 @@ +{ + "pytest-terraform": 1, + "outputs": {}, + "resources": { + "aws_ami": { + "amazon_linux": { + "architecture": "x86_64", + "arn": "arn:aws:ec2:us-east-1::image/ami-0d7734f53f491186b", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": { + "delete_on_termination": "true", + "encrypted": "false", + "iops": "0", + "snapshot_id": "snap-00e9c002a992550a7", + "throughput": "0", + "volume_size": "8", + "volume_type": "gp2" + }, + "no_device": "", + "virtual_name": "" + } + ], + "boot_mode": "", + "creation_date": "2022-07-16T02:39:01.000Z", + "deprecation_time": "2024-07-16T02:39:01.000Z", + "description": "Amazon Linux AMI 2018.03.0.20220705.1 x86_64 HVM gp2", + "ena_support": true, + "executable_users": null, + "filter": [ + { + "name": "name", + "values": [ + "amzn-ami-hvm-*-x86_64-gp2" + ] + } + ], + "hypervisor": "xen", + "id": "ami-0d7734f53f491186b", + "image_id": "ami-0d7734f53f491186b", + "image_location": "amazon/amzn-ami-hvm-2018.03.0.20220705.1-x86_64-gp2", + "image_owner_alias": "amazon", + "image_type": "machine", + "include_deprecated": false, + "kernel_id": "", + "most_recent": true, + "name": "amzn-ami-hvm-2018.03.0.20220705.1-x86_64-gp2", + "name_regex": null, + "owner_id": "644160558196", + "owners": [ + "amazon" + ], + "platform": "", + "platform_details": "Linux/UNIX", + "product_codes": [], + "public": true, + "ramdisk_id": "", + "root_device_name": "/dev/xvda", + "root_device_type": "ebs", + "root_snapshot_id": "snap-00e9c002a992550a7", + "sriov_net_support": "simple", + "state": "available", + "state_reason": { + "code": "UNSET", + "message": "UNSET" + }, + "tags": {}, + "tpm_support": "", + "usage_operation": "RunInstances", + "virtualization_type": "hvm" + } + }, + "aws_instance": { + "no_protection": { + "ami": "ami-0d7734f53f491186b", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-0daf36c03bf9c5e75", + "associate_public_ip_address": false, + "availability_zone": "us-east-1f", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 1, + "cpu_threads_per_core": 1, + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": false, + "disable_api_termination": false, + "ebs_block_device": [], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "", + "id": "i-0daf36c03bf9c5e75", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-0d3a0342e5b7b9403", + "private_dns": "ip-10-0-1-15.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.15", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 100, + "kms_key_id": "", + "tags": {}, + "throughput": 0, + "volume_id": "vol-07fccc8f6c75d1d1c", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-0e172a55c20169303", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-09c6a03dec497a6b0" + ] + }, + "stop_protection": { + "ami": "ami-0d7734f53f491186b", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-0ecfe7c50cf2f76b2", + "associate_public_ip_address": false, + "availability_zone": "us-east-1f", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 1, + "cpu_threads_per_core": 1, + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": true, + "disable_api_termination": false, + "ebs_block_device": [], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "", + "id": "i-0ecfe7c50cf2f76b2", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-0d6d13c72ac812cae", + "private_dns": "ip-10-0-1-124.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.124", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 100, + "kms_key_id": "", + "tags": {}, + "throughput": 0, + "volume_id": "vol-0d2b762057cf67207", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-0e172a55c20169303", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-09c6a03dec497a6b0" + ] + }, + "termination_protection": { + "ami": "ami-0d7734f53f491186b", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-017e55c722d539f90", + "associate_public_ip_address": false, + "availability_zone": "us-east-1f", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 1, + "cpu_threads_per_core": 1, + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": false, + "disable_api_termination": true, + "ebs_block_device": [], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "", + "id": "i-017e55c722d539f90", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-004e46d428e6b695a", + "private_dns": "ip-10-0-1-168.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.168", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 100, + "kms_key_id": "", + "tags": {}, + "throughput": 0, + "volume_id": "vol-0f1ee2360fa73f764", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-0e172a55c20169303", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-09c6a03dec497a6b0" + ] + } + }, + "aws_subnet": { + "example": { + "arn": "arn:aws:ec2:us-east-1:644160558196:subnet/subnet-0e172a55c20169303", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-1f", + "availability_zone_id": "use1-az5", + "cidr_block": "10.0.1.0/24", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-0e172a55c20169303", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_native": false, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "644160558196", + "private_dns_hostname_type_on_launch": "ip-name", + "tags": null, + "tags_all": {}, + "timeouts": null, + "vpc_id": "vpc-00255c1f52c0a88d6" + } + }, + "aws_vpc": { + "example": { + "arn": "arn:aws:ec2:us-east-1:644160558196:vpc/vpc-00255c1f52c0a88d6", + "assign_generated_ipv6_cidr_block": false, + "cidr_block": "10.0.0.0/16", + "default_network_acl_id": "acl-00281436354a0a4f2", + "default_route_table_id": "rtb-0c7d1e90e3c8ce222", + "default_security_group_id": "sg-09c6a03dec497a6b0", + "dhcp_options_id": "dopt-0965473f41c19ccbd", + "enable_classiclink": false, + "enable_classiclink_dns_support": false, + "enable_dns_hostnames": false, + "enable_dns_support": true, + "id": "vpc-00255c1f52c0a88d6", + "instance_tenancy": "default", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_association_id": "", + "ipv6_cidr_block": "", + "ipv6_cidr_block_network_border_group": "", + "ipv6_ipam_pool_id": "", + "ipv6_netmask_length": 0, + "main_route_table_id": "rtb-0c7d1e90e3c8ce222", + "owner_id": "644160558196", + "tags": null, + "tags_all": {} + } + } + } +} \ No newline at end of file diff --git a/tests/test_ec2.py b/tests/test_ec2.py index 0d9bf8dee05..05d949bfe86 100644 --- a/tests/test_ec2.py +++ b/tests/test_ec2.py @@ -17,6 +17,117 @@ from .common import BaseTest +import pytest +from pytest_terraform import terraform + + +@terraform('ec2_stop_protection_enabled') +def test_ec2_stop_protection_enabled(test, ec2_stop_protection_enabled): + aws_region = 'us-east-1' + session_factory = test.replay_flight_data('ec2_stop_protection_enabled', region=aws_region) + + p = test.load_policy( + { + 'name': 'ec2_stop_protection_enabled', + 'resource': 'ec2', + 'filters': [ + {'State.Name': 'running'}, + {'type': 'stop-protected'}, + ], + }, + session_factory=session_factory, + config={'region': aws_region}, + ) + + resources = p.run() + test.assertEqual(len(resources), 1) + test.assertEqual( + resources[0]['InstanceId'], + ec2_stop_protection_enabled['aws_instance.stop_protection.id']) + + +@terraform('ec2_stop_protection_disabled') +def test_ec2_stop_protection_disabled(test, ec2_stop_protection_disabled): + aws_region = 'us-east-1' + session_factory = test.replay_flight_data('ec2_stop_protection_disabled', region=aws_region) + + p = test.load_policy( + { + 'name': 'ec2_stop_protection_disabled', + 'resource': 'ec2', + 'filters': [ + {'State.Name': 'running'}, + {'not': [{'type': 'stop-protected'}]}, + ], + }, + session_factory=session_factory, + config={'region': aws_region}, + ) + + resources = p.run() + test.assertEqual(len(resources), 2) + + resource_ids = [i['InstanceId'] for i in resources] + test.assertIn( + ec2_stop_protection_disabled['aws_instance.termination_protection.id'], + resource_ids) + test.assertIn( + ec2_stop_protection_disabled['aws_instance.no_protection.id'], + resource_ids) + + +def test_ec2_stop_protection_filter_permissions(test): + policy = test.load_policy( + { + 'name': 'ec2-stop-protection', + 'resource': 'ec2', + 'filters': [{'type': 'stop-protected'}], + }, + ) + permissions = policy.get_permissions() + test.assertEqual( + permissions, + { + 'ec2:DescribeInstances', + 'ec2:DescribeTags', + 'ec2:DescribeInstanceAttribute', + }, + ) + + +@pytest.mark.parametrize( + 'botocore_version', + ['1.26.6', '1.25.8', '0.27.27'] +) +def test_ec2_stop_protection_lower_botocore_version_validation(test, botocore_version): + with mock.patch('botocore.__version__', botocore_version): + with test.assertRaises(PolicyValidationError) as cm: + policy = test.load_policy( + { + 'name': 'ec2-stop-protection', + 'resource': 'ec2', + 'filters': [{'type': 'stop-protected'}], + }, + ) + policy.validate() + test.assertIn('requires botocore version 1.26.7 or above', str(cm.exception)) + + +@pytest.mark.parametrize( + 'botocore_version', + ['1.26.7', '1.26.8', '1.27.0', '2.0.0'] +) +def test_ec2_stop_protection_above_botocore_version_validation(test, botocore_version): + with mock.patch('botocore.__version__', botocore_version): + policy = test.load_policy( + { + 'name': 'ec2-stop-protection', + 'resource': 'ec2', + 'filters': [{'type': 'stop-protected'}], + }, + ) + policy.validate() + class TestEc2NetworkLocation(BaseTest): def test_ec2_network_location_terminated(self): From ce769a180c5162a9989c3928c7c5fcfb2eeee055 Mon Sep 17 00:00:00 2001 From: Thanos222 <111351162+Thanos222@users.noreply.github.com> Date: Thu, 18 Aug 2022 16:05:16 +0200 Subject: [PATCH 2/3] tools/c7n_logexporter - allow user specified role for put_subscription_filter (#7657) --- tools/c7n_logexporter/README.md | 2 ++ .../c7n_logexporter/exporter.py | 23 +++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tools/c7n_logexporter/README.md b/tools/c7n_logexporter/README.md index 568bd244348..6032367918e 100644 --- a/tools/c7n_logexporter/README.md +++ b/tools/c7n_logexporter/README.md @@ -75,6 +75,8 @@ destination: accounts: - name: custodian-demo + # https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CreateSubscriptionFilter-IAMrole.html + subscription-role: "arn:aws:iam::111111111111:role/" role: "arn:aws:iam::111111111111:role/CloudCustodianRole" groups: - "/aws/lambda/*" diff --git a/tools/c7n_logexporter/c7n_logexporter/exporter.py b/tools/c7n_logexporter/c7n_logexporter/exporter.py index 0ac968f8d84..a95389105a4 100644 --- a/tools/c7n_logexporter/c7n_logexporter/exporter.py +++ b/tools/c7n_logexporter/c7n_logexporter/exporter.py @@ -62,6 +62,7 @@ 'required': ['role', 'groups'], 'properties': { 'name': {'type': 'string'}, + 'subscription-role': {'type': 'string'}, 'role': {'oneOf': [ {'type': 'array', 'items': {'type': 'string'}}, {'type': 'string'}]}, @@ -130,7 +131,7 @@ def validate(config): return data -def _process_subscribe_group(client, group_name, subscription, distribution): +def _process_subscribe_group(client, group_name, subscription, distribution, role_arn): sub_name = subscription.get('name', 'FlowLogStream') filters = client.describe_subscription_filters( logGroupName=group_name).get('subscriptionFilters', ()) @@ -143,12 +144,15 @@ def _process_subscribe_group(client, group_name, subscription, distribution): else: client.delete_subscription_filter( logGroupName=group_name, filterName=sub_name) - client.put_subscription_filter( - logGroupName=group_name, - destinationArn=subscription['destination-arn'], - filterName=sub_name, - filterPattern="", - distribution=distribution) + kwargs = { + 'logGroupName': group_name, + 'destinationArn': subscription['destination-arn'], + 'filterName': sub_name, + 'filterPattern': "", + 'distribution': distribution, + 'roleArn': role_arn + } + client.put_subscription_filter(**{k: v for k, v in kwargs.items() if v is not None}) @cli.command() @@ -204,6 +208,7 @@ def subscribe_account(t_account, subscription, region): session = get_session(t_account['role'], region) client = session.client('logs') distribution = subscription.get('distribution', 'ByLogStream') + role_arn = account.get('subscription-role') for g in account.get('groups'): if (g.endswith('*')): @@ -212,9 +217,9 @@ def subscribe_account(t_account, subscription, region): allLogGroups = paginator.paginate(logGroupNamePrefix=g).build_full_result() for l in allLogGroups['logGroups']: _process_subscribe_group( - client, l['logGroupName'], subscription, distribution) + client, l['logGroupName'], subscription, distribution, role_arn) else: - _process_subscribe_group(client, g, subscription, distribution) + _process_subscribe_group(client, g, subscription, distribution, role_arn) if subscription.get('managed-policy'): if subscription.get('destination-role'): From f62f6b96a594c3800b0cddf0228c4ff81fe4fb96 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Fri, 19 Aug 2022 10:07:18 -0400 Subject: [PATCH 3/3] core - sqlkv cache file (#7659) --- c7n/cache.py | 137 ++++++++++++------- c7n/policy.py | 3 +- c7n/query.py | 3 + c7n/resources/iam.py | 33 +++-- poetry.lock | 35 +++-- pyproject.toml | 1 + tests/test_cache.py | 193 ++++++++++----------------- tests/test_iam.py | 9 +- tools/c7n_azure/c7n_azure/filters.py | 1 - tools/c7n_azure/c7n_azure/query.py | 3 +- tools/c7n_gcp/c7n_gcp/query.py | 26 +++- 11 files changed, 227 insertions(+), 217 deletions(-) diff --git a/c7n/cache.py b/c7n/cache.py index aa75e0a2f51..8da1e80aa71 100644 --- a/c7n/cache.py +++ b/c7n/cache.py @@ -5,9 +5,10 @@ """ import pickle # nosec nosemgrep +from datetime import datetime, timedelta import os import logging -import time +import sqlite3 log = logging.getLogger('custodian.cache') @@ -30,12 +31,11 @@ def factory(config): if not CACHE_NOTIFY: log.debug("Using in-memory cache") CACHE_NOTIFY = True - return InMemoryCache() + return InMemoryCache(config) + return SqlKvCache(config) - return FileCacheManager(config) - -class NullCache: +class Cache: def __init__(self, config): self.config = config @@ -52,74 +52,115 @@ def save(self, key, data): def size(self): return 0 + def close(self): + pass + + def __enter__(self): + self.load() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + -class InMemoryCache: +class NullCache(Cache): + pass + + +class InMemoryCache(Cache): # Running in a temporary environment, so keep as a cache. __shared_state = {} - def __init__(self): + def __init__(self, config): + super().__init__(config) self.data = self.__shared_state def load(self): return True def get(self, key): - return self.data.get(pickle.dumps(key)) # nosemgrep + return self.data.get(encode(key)) def save(self, key, data): - self.data[pickle.dumps(key)] = data # nosemgrep + self.data[encode(key)] = data def size(self): return sum(map(len, self.data.values())) -class FileCacheManager: +def encode(key): + return pickle.dumps(key, protocol=pickle.HIGHEST_PROTOCOL) # nosemgrep + + +def resolve_path(path): + return os.path.abspath( + os.path.expanduser( + os.path.expandvars(path))) + + +class SqlKvCache(Cache): + + create_table = """ + create table if not exists c7n_cache ( + key blob primary key, + value blob, + create_date timestamp + ) + """ def __init__(self, config): - self.config = config + super().__init__(config) self.cache_period = config.cache_period - self.cache_path = os.path.abspath( - os.path.expanduser( - os.path.expandvars( - config.cache))) - self.data = {} - - def get(self, key): - k = pickle.dumps(key) # nosemgrep - return self.data.get(k) + self.cache_path = resolve_path(config.cache) + self.conn = None def load(self): - if self.data: - return True - if os.path.isfile(self.cache_path): - if (time.time() - os.stat(self.cache_path).st_mtime > - self.config.cache_period * 60): - return False + # migration from pickle cache file + if os.path.exists(self.cache_path): with open(self.cache_path, 'rb') as fh: - try: - self.data = pickle.load(fh) # nosec nosemgrep - except EOFError: - return False - log.debug("Using cache file %s" % self.cache_path) - return True + header = fh.read(15) + if header != b'SQLite format 3': + log.debug('removing old cache file') + os.remove(self.cache_path) + elif not os.path.exists(os.path.dirname(self.cache_path)): + # parent directory creation + os.makedirs(os.path.dirname(self.cache_path)) + self.conn = sqlite3.connect(self.cache_path) + self.conn.execute(self.create_table) + with self.conn as cursor: + log.debug('expiring stale cache entries') + cursor.execute( + 'delete from c7n_cache where create_date < ?', + [datetime.utcnow() - timedelta(minutes=self.cache_period)]) + return True - def save(self, key, data): - try: - with open(self.cache_path, 'wb') as fh: # nosec - self.data[pickle.dumps(key)] = data # nosemgrep - pickle.dump(self.data, fh, protocol=2) # nosemgrep - except Exception as e: - log.warning("Could not save cache %s err: %s" % ( - self.cache_path, e)) - if not os.path.exists(self.cache_path): - directory = os.path.dirname(self.cache_path) - log.info('Generating Cache directory: %s.' % directory) - try: - os.makedirs(directory) - except Exception as e: - log.warning("Could not create directory: %s err: %s" % ( - directory, e)) + def get(self, key): + with self.conn as cursor: + r = cursor.execute( + 'select value, create_date from c7n_cache where key = ?', + [sqlite3.Binary(encode(key))] + ) + row = r.fetchone() + if row is None: + return None + value, create_date = row + create_date = sqlite3.converters['TIMESTAMP'](create_date.encode('utf8')) + if (datetime.utcnow() - create_date).total_seconds() / 60.0 > self.cache_period: + return None + return pickle.loads(value) # nosec nosemgrep + + def save(self, key, data, timestamp=None): + with self.conn as cursor: + timestamp = timestamp or datetime.utcnow() + cursor.execute( + 'replace into c7n_cache (key, value, create_date) values (?, ?, ?)', + (sqlite3.Binary(encode(key)), sqlite3.Binary(encode(data)), timestamp)) def size(self): return os.path.exists(self.cache_path) and os.path.getsize(self.cache_path) or 0 + + def close(self): + if self.conn: + self.conn.close() + self.conn = None diff --git a/c7n/policy.py b/c7n/policy.py index 3093925c029..fbeb8d9fe15 100644 --- a/c7n/policy.py +++ b/c7n/policy.py @@ -1285,8 +1285,7 @@ def __call__(self): resources = mode.provision() else: resources = mode.run() - # clear out resource manager post run, to clear cache - self.resource_manager = self.load_resource_manager() + return resources run = __call__ diff --git a/c7n/query.py b/c7n/query.py index d16e25c3875..da69c0166d5 100644 --- a/c7n/query.py +++ b/c7n/query.py @@ -528,6 +528,8 @@ def resources(self, query=None, augment=True) -> List[dict]: # Don't pollute cache with unaugmented resources. self._cache.save(cache_key, resources) + self._cache.close() + resource_count = len(resources) with self.ctx.tracer.subsegment('filter'): resources = self.filter_resources(resources) @@ -556,6 +558,7 @@ def _get_cached_resources(self, ids): m = self.get_model() id_set = set(ids) return [r for r in resources if r[m.id] in id_set] + self._cache.close() return None def get_resources(self, ids, cache=True, augment=True): diff --git a/c7n/resources/iam.py b/c7n/resources/iam.py index 41f1ee08e88..571528a2db8 100644 --- a/c7n/resources/iam.py +++ b/c7n/resources/iam.py @@ -1605,20 +1605,25 @@ def get_value_or_schema_default(self, k): return self.schema['properties'][k]['default'] def get_credential_report(self): - report = self.manager._cache.get('iam-credential-report') - if report: - return report - data = self.fetch_credential_report() - report = {} - if isinstance(data, bytes): - reader = csv.reader(io.StringIO(data.decode('utf-8'))) - else: - reader = csv.reader(io.StringIO(data)) - headers = next(reader) - for line in reader: - info = dict(zip(headers, line)) - report[info['user']] = self.process_user_record(info) - self.manager._cache.save('iam-credential-report', report) + cache = self.manager._cache + with cache: + cache_key = {'account': self.manager.config.account_id, 'iam-credential-report': True} + report = cache.get(cache_key) + + if report: + return report + data = self.fetch_credential_report() + report = {} + if isinstance(data, bytes): + reader = csv.reader(io.StringIO(data.decode('utf-8'))) + else: + reader = csv.reader(io.StringIO(data)) + headers = next(reader) + for line in reader: + info = dict(zip(headers, line)) + report[info['user']] = self.process_user_record(info) + cache.save(cache_key, report) + return report @classmethod diff --git a/poetry.lock b/poetry.lock index bd952b33df9..b5fde0ac0e3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -207,6 +207,17 @@ mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.7.0,<2.8.0" pyflakes = ">=2.3.0,<2.4.0" +[[package]] +name = "freezegun" +version = "1.2.2" +description = "Let your Python tests travel through time" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +python-dateutil = ">=2.7" + [[package]] name = "idna" version = "3.3" @@ -863,7 +874,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "49cb11b477adf87c04c9e56e4694aabf5afa53f1870ad23aa793dfc8bdd739c7" +content-hash = "79a1d8fb3eaae9c62d2125f77cd78d01fd668b8da0f2295c3f0705cb8fda9a8d" [metadata.files] argcomplete = [ @@ -876,20 +887,14 @@ aws-xray-sdk = [] bleach = [] boto3 = [] botocore = [] -certifi = [ - {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, - {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, -] +certifi = [] cffi = [] charset-normalizer = [] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] -colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, -] +colorama = [] coverage = [ {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"}, {file = "coverage-5.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b"}, @@ -957,14 +962,12 @@ flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] +freezegun = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -importlib-metadata = [ - {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, - {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, -] +importlib-metadata = [] importlib-resources = [] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -1257,11 +1260,7 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -tabulate = [ - {file = "tabulate-0.8.10-py3-none-any.whl", hash = "sha256:0ba055423dbaa164b9e456abe7920c5e8ed33fcc16f6d1b2f2d152c8e1e8b4fc"}, - {file = "tabulate-0.8.10-py3.8.egg", hash = "sha256:436f1c768b424654fce8597290d2764def1eea6a77cfa5c33be00b1bc0f4f63d"}, - {file = "tabulate-0.8.10.tar.gz", hash = "sha256:6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519"}, -] +tabulate = [] termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, ] diff --git a/pyproject.toml b/pyproject.toml index 1bc62c4f623..f83943f1d9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ psutil = "^5.7.0" twine = "^3.1.1" pytest-sugar = "^0.9.2" click = "^8.0" +freezegun = "^1.2.2" [tool.black] skip-string-normalization = true diff --git a/tests/test_cache.py b/tests/test_cache.py index e7524cb790a..3418d864fe7 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,12 +1,16 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 -from unittest import TestCase -from c7n import cache, config from argparse import Namespace -import pickle -import tempfile -import mock +from datetime import datetime, timedelta import os +import pickle +import sqlite3 +import sys +from unittest import TestCase + +import pytest + +from c7n import cache, config class TestCache(TestCase): @@ -14,7 +18,7 @@ class TestCache(TestCase): def test_factory(self): self.assertIsInstance(cache.factory(None), cache.NullCache) test_config = Namespace(cache_period=60, cache="test-cloud-custodian.cache") - self.assertIsInstance(cache.factory(test_config), cache.FileCacheManager) + self.assertIsInstance(cache.factory(test_config), cache.SqlKvCache) test_config.cache = None self.assertIsInstance(cache.factory(test_config), cache.NullCache) @@ -27,126 +31,71 @@ def test_mem_factory(self): cache.InMemoryCache) def test_get_set(self): - mem_cache = cache.InMemoryCache() + mem_cache = cache.InMemoryCache({}) mem_cache.save({'region': 'us-east-1'}, {'hello': 'world'}) self.assertEqual(mem_cache.size(), 1) self.assertEqual(mem_cache.load(), True) - mem_cache = cache.InMemoryCache() + mem_cache = cache.InMemoryCache({}) self.assertEqual( mem_cache.get({'region': 'us-east-1'}), {'hello': 'world'}) - - -class FileCacheManagerTest(TestCase): - - def setUp(self): - self.test_config = Namespace( - cache_period=60, cache="test-cloud-custodian.cache" - ) - self.test_cache = cache.FileCacheManager(self.test_config) - self.test_key = "test" - self.bad_key = "bad" - self.test_value = [1, 2, 3] - - def test_get_set(self): - t = self.temporary_file_with_cleanup() - c = cache.FileCacheManager(Namespace(cache_period=60, cache=t.name)) - self.assertFalse(c.load()) - k1 = {"account": "12345678901234", "region": "us-west-2", "resource": "ec2"} - c.save(k1, range(5)) - self.assertEqual(c.get(k1), range(5)) - k2 = {"account": "98765432101234", "region": "eu-west-1", "resource": "asg"} - c.save(k2, range(2)) - self.assertEqual(c.get(k1), range(5)) - self.assertEqual(c.get(k2), range(2)) - - c2 = cache.FileCacheManager(Namespace(cache_period=60, cache=t.name)) - self.assertTrue(c2.load()) - self.assertEqual(c2.get(k1), range(5)) - self.assertEqual(c2.get(k2), range(2)) - - def test_get(self): - # mock the pick and set it to the data variable - test_pickle = pickle.dumps( - {pickle.dumps(self.test_key): self.test_value}, protocol=2 - ) - self.test_cache.data = pickle.loads(test_pickle) - - # assert - self.assertEqual(self.test_cache.get(self.test_key), self.test_value) - self.assertEqual(self.test_cache.get(self.bad_key), None) - - def test_load(self): - t = self.temporary_file_with_cleanup(suffix=".cache") - - load_config = Namespace(cache_period=0, cache=t.name) - load_cache = cache.FileCacheManager(load_config) - self.assertFalse(load_cache.load()) - load_cache.data = {"key": "value"} - self.assertTrue(load_cache.load()) - - @mock.patch.object(cache.os, "makedirs") - @mock.patch.object(cache.os.path, "exists") - @mock.patch.object(cache.pickle, "dump") - @mock.patch.object(cache.pickle, "dumps") - def test_save_exists(self, mock_dumps, mock_dump, mock_exists, mock_mkdir): - # path exists then we dont need to create the folder - mock_exists.return_value = True - # tempfile to hold the pickle - temp_cache_file = self.temporary_file_with_cleanup() - - self.test_cache.cache_path = temp_cache_file.name - # make the call - self.test_cache.save(self.test_key, self.test_value) - - # assert if file already exists - self.assertFalse(mock_mkdir.called) - self.assertTrue(mock_dumps.called) - self.assertTrue(mock_dump.called) - - # mkdir should NOT be called, but pickles should - self.assertEqual(mock_mkdir.call_count, 0) - self.assertEqual(mock_dump.call_count, 1) - self.assertEqual(mock_dumps.call_count, 1) - - @mock.patch.object(cache.os, "makedirs") - @mock.patch.object(cache.os.path, "exists") - @mock.patch.object(cache.pickle, "dump") - @mock.patch.object(cache.pickle, "dumps") - def test_save_doesnt_exists(self, mock_dumps, mock_dump, mock_exists, mock_mkdir): - temp_cache_file = self.temporary_file_with_cleanup() - - self.test_cache.cache_path = temp_cache_file.name - - # path doesnt exists then we will create the folder - # raise some sort of exception in the try - mock_exists.return_value = False - mock_dump.side_effect = Exception("Error") - mock_mkdir.side_effect = Exception("Error") - - # make the call - self.test_cache.save(self.test_key, self.test_value) - - # assert if file doesnt exists - self.assertTrue(mock_mkdir.called) - self.assertTrue(mock_dumps.called) - self.assertTrue(mock_dump.called) - - # all 3 should be called once - self.assertEqual(mock_mkdir.call_count, 1) - self.assertEqual(mock_dump.call_count, 1) - self.assertEqual(mock_dumps.call_count, 1) - - def temporary_file_with_cleanup(self, **kwargs): - """ - NamedTemporaryFile with delete=True has - significantly different behavior on Windows - so we utilize delete=False to simplify maintaining - compatibility. - """ - t = tempfile.NamedTemporaryFile(delete=False, **kwargs) - - self.addCleanup(os.unlink, t.name) - self.addCleanup(t.close) - return t + mem_cache.close() + + +def test_sqlkv(tmp_path): + kv = cache.SqlKvCache(config.Bag(cache=tmp_path / "cache.db", cache_period=60)) + kv.load() + k1 = {"account": "12345678901234", "region": "us-west-2", "resource": "ec2"} + v1 = [{'id': 'a'}, {'id': 'b'}] + + assert kv.get(k1) is None + kv.save(k1, v1) + assert kv.get(k1) == v1 + kv.close() + + +def test_sqlkv_get_expired(tmp_path): + kv = cache.SqlKvCache(config.Bag(cache=tmp_path / "cache.db", cache_period=60)) + kv.load() + kv1 = {'a': 'b', 'c': 'd'} + kv.save(kv1, kv1, datetime.utcnow() - timedelta(days=10)) + assert kv.get(kv1) is None + + +def test_sqlkv_load_gc(tmp_path): + kv = cache.SqlKvCache(config.Bag(cache=tmp_path / "cache.db", cache_period=60)) + + # seed old values with manual connection + kv.conn = sqlite3.connect(kv.cache_path) + kv.conn.execute(kv.create_table) + kv1 = {'a': 'b', 'c': 'd'} + kv2 = {'b': 'a', 'd': 'c'} + kv.save(kv1, kv1, datetime.utcnow() - timedelta(days=10)) + kv.save(kv2, kv2, datetime.utcnow() - timedelta(minutes=5)) + + kv.load() + assert kv.get(kv1) is None + assert kv.get(kv2) == kv2 + + +def test_sqlkv_parent_dir_create(tmp_path): + cache_path = tmp_path / ".cache" / "cache.db" + kv = cache.SqlKvCache(config.Bag(cache=cache_path, cache_period=60)) + kv.load() + assert os.path.exists(os.path.dirname(cache_path)) + + +@pytest.mark.skipif( + sys.platform == 'win32', + reason="windows can't remove a recently created but closed file") +def test_sqlkv_convert(tmp_path): + cache_path = tmp_path / "cache2.db" + with open(cache_path, 'wb') as fh: + pickle.dump({'kv': 'abc'}, fh) + fh.close() + kv = cache.SqlKvCache(config.Bag(cache=cache_path, cache_period=60)) + kv.load() + kv.close() + with open(cache_path, 'rb') as fh: + assert fh.read(15) == b"SQLite format 3" diff --git a/tests/test_iam.py b/tests/test_iam.py index 7e830b775ef..412c729e25a 100644 --- a/tests/test_iam.py +++ b/tests/test_iam.py @@ -1,7 +1,6 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 import json -import datetime import os import mock import tempfile @@ -9,8 +8,8 @@ from unittest import TestCase from .common import load_data, BaseTest, functional -from .test_offhours import mock_datetime_now +import freezegun import pytest from pytest_terraform import terraform from dateutil import parser @@ -120,7 +119,7 @@ def test_credential_access_key_multifilter_delete(self): 'matched': True}]}, session_factory=factory) - with mock_datetime_now(parser.parse("2020-01-01"), datetime): + with freezegun.freeze_time('2020-01-01'): resources = p.run() self.assertEqual(len(resources), 1) self.assertEqual(len(resources[0]['c7n:matched-keys']), 1) @@ -240,7 +239,7 @@ def test_access_key_last_service(self): session_factory=session_factory, cache=True, ) - with mock_datetime_now(parser.parse("2016-11-25T20:27:00+00:00"), datetime): + with freezegun.freeze_time("2016-11-25T20:27"): resources = p.run() self.assertEqual(len(resources), 1) self.assertEqual(sorted([r["UserName"] for r in resources]), ["kapil"]) @@ -271,7 +270,7 @@ def test_old_console_users(self): cache=True, ) - with mock_datetime_now(parser.parse("2016-11-25T20:27:00+00:00"), datetime): + with freezegun.freeze_time("2016-11-25T20:27:00+00:00"): resources = p.run() self.assertEqual(len(resources), 3) self.assertEqual( diff --git a/tools/c7n_azure/c7n_azure/filters.py b/tools/c7n_azure/c7n_azure/filters.py index ea9484b2f74..a23a3a551f3 100644 --- a/tools/c7n_azure/c7n_azure/filters.py +++ b/tools/c7n_azure/c7n_azure/filters.py @@ -1012,6 +1012,5 @@ def __init__(self, data, manager=None): def process(self, resources, event=None): parent_resources = self.parent_filter.process(self.parent_manager.resources()) parent_resources_ids = [p['id'] for p in parent_resources] - parent_key = self.manager.resource_type.parent_key return [r for r in resources if r[parent_key] in parent_resources_ids] diff --git a/tools/c7n_azure/c7n_azure/query.py b/tools/c7n_azure/c7n_azure/query.py index b8186047a1b..091c4928109 100644 --- a/tools/c7n_azure/c7n_azure/query.py +++ b/tools/c7n_azure/c7n_azure/query.py @@ -159,7 +159,6 @@ def filter(self, resource_manager, **params): .format(parent[parents.resource_type.id], e)) if m.raise_on_exception: raise e - return results @@ -282,6 +281,8 @@ def resources(self, query=None, augment=True): resources = self.augment(resources) self._cache.save(cache_key, resources) + self._cache.close() + with self.ctx.tracer.subsegment('filter'): resource_count = len(resources) resources = self.filter_resources(resources) diff --git a/tools/c7n_gcp/c7n_gcp/query.py b/tools/c7n_gcp/c7n_gcp/query.py index 689d79db97c..e436962726d 100644 --- a/tools/c7n_gcp/c7n_gcp/query.py +++ b/tools/c7n_gcp/c7n_gcp/query.py @@ -188,14 +188,28 @@ def get_resource_query(self): def resources(self, query=None): q = query or self.get_resource_query() - key = self.get_cache_key(q) - resources = self._fetch_resources(q) - self._cache.save(key, resources) - + cache_key = self.get_cache_key(q) + resources = None + + if self._cache.load(): + resources = self._cache.get(cache_key) + if resources is not None: + self.log.debug("Using cached %s: %d" % ( + "%s.%s" % (self.__class__.__module__, + self.__class__.__name__), + len(resources))) + + if resources is None: + with self.ctx.tracer.subsegment('resource-fetch'): + resources = self._fetch_resources(q) + self._cache.save(cache_key, resources) + + self._cache.close() resource_count = len(resources) - resources = self.filter_resources(resources) + with self.ctx.tracer.subsegment('filter'): + resources = self.filter_resources(resources) - # Check if we're out of a policies execution limits. + # Check resource limits if we're the current policy execution. if self.data == self.ctx.policy.data: self.check_resource_limit(len(resources), resource_count) return resources