From 02ec5d7d6bdbbef875db963f325f0e8a2fa6f72f Mon Sep 17 00:00:00 2001 From: Michael McMurray Date: Fri, 13 Jan 2023 12:32:29 -0500 Subject: [PATCH 1/6] aws - subnet - ip application filter --- c7n/resources/vpc.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py index 39c550ac165..2f44711ce92 100644 --- a/c7n/resources/vpc.py +++ b/c7n/resources/vpc.py @@ -2885,3 +2885,64 @@ def process(self, resources, event=None): results.append(resource) return results + + +@Subnet.filter_registry.register('ip-allocation-threshold') +class SubnetIpAllocationFilter(Filter): + """Filters subnets based on ip allocation percentage + + :example: + + .. code-block:: yaml + + policies: + - name: subnet-ip-threshold-policy + resource: subnet + filters: + - type: ip-allocation-threshold + percentage: 80 + op: gte + """ + schema = type_schema( + 'ip-allocation-threshold', + percentage={'type': 'number'}, + op={'enum': ['eq', 'ne', 'lt', 'gt', 'lte', 'gte']} + ) + + permissions = ("ec2:DescribeSubnets",) + + def calculate_ip_allocation(self, subnet): + subnetMask = subnet.get('CidrBlock').split('/')[1] + hostBits = 32 - int(subnetMask) + totalHost = ((2 ** hostBits) - 2) + availableHost = subnet.get('AvailableIpAddressCount') + ipsUsed = totalHost - availableHost + percentageOfIpsUsed = (ipsUsed / totalHost) * 100 + return percentageOfIpsUsed + + def process(self, resources, event=None): + results = [] + threshold_percentage = self.data.get('percentage') + op = self.data.get('op') + for subnet in resources: + percentage_used = self.calculate_ip_allocation(subnet) + match op: + case 'eq': + if threshold_percentage == percentage_used: + results.append(subnet) + case 'ne': + if threshold_percentage != percentage_used: + results.append(subnet) + case 'lt': + if percentage_used < threshold_percentage: + results.append(subnet) + case 'gt': + if percentage_used > threshold_percentage: + results.append(subnet) + case 'lte': + if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + case 'gte': + if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + return results From 2c167fad52c64f52721762ab5ff94d6a84209793 Mon Sep 17 00:00:00 2001 From: Gianncarlo Giannattasio <89796775+CarloGiannattasio@users.noreply.github.com> Date: Mon, 23 Jan 2023 11:15:52 -0500 Subject: [PATCH 2/6] vpc.py - ip-allocation-threshold - updated to work with python 3.7 --- c7n/resources/vpc.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py index 2f44711ce92..aabb7d12319 100644 --- a/c7n/resources/vpc.py +++ b/c7n/resources/vpc.py @@ -2926,23 +2926,22 @@ def process(self, resources, event=None): op = self.data.get('op') for subnet in resources: percentage_used = self.calculate_ip_allocation(subnet) - match op: - case 'eq': - if threshold_percentage == percentage_used: - results.append(subnet) - case 'ne': - if threshold_percentage != percentage_used: - results.append(subnet) - case 'lt': - if percentage_used < threshold_percentage: - results.append(subnet) - case 'gt': - if percentage_used > threshold_percentage: - results.append(subnet) - case 'lte': - if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): - results.append(subnet) - case 'gte': - if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): - results.append(subnet) + if op == 'eq': + if threshold_percentage == percentage_used: + results.append(subnet) + elif op == 'ne': + if threshold_percentage != percentage_used: + results.append(subnet) + elif op == 'lt': + if percentage_used < threshold_percentage: + results.append(subnet) + elif op == 'gt': + if percentage_used > threshold_percentage: + results.append(subnet) + elif op == 'lte': + if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + elif op == 'gte': + if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) return results From 15e1286b7ffd24b17fdf9c728a552e48368ca33d Mon Sep 17 00:00:00 2001 From: Gianncarlo Giannattasio Date: Fri, 20 Oct 2023 10:22:13 -0400 Subject: [PATCH 3/6] update API logic --- Pipfile | 84 +++++++++++++++++ c7n/manager.py | 16 +++- c7n/resource_metadata_update_with_email.py | 102 +++++++++++++++++++++ c7n/resources/asg.py | 3 +- c7n/resources/rds.py | 3 +- c7n/resources/vpc.py | 60 ------------ tests/conftest.py | 2 +- 7 files changed, 204 insertions(+), 66 deletions(-) create mode 100644 Pipfile create mode 100644 c7n/resource_metadata_update_with_email.py diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000000..0cd254c597f --- /dev/null +++ b/Pipfile @@ -0,0 +1,84 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +argcomplete = "==2.0.0" +attrs = "==22.1.0" +aws-xray-sdk = "==2.11.0" +bleach = "==5.0.1" +boto3 = "==1.26.30" +botocore = "==1.29.30" +certifi = "==2022.12.7" +cffi = "==1.15.1" +charset-normalizer = "==2.0.12" +click = "==8.1.3" +colorama = "==0.4.6" +coverage = {version = "==6.5.0", extras = ["toml"]} +cryptography = "==38.0.4" +docutils = "==0.17.1" +exceptiongroup = "==1.0.4" +execnet = "==1.9.0" +flake8 = "==3.9.2" +freezegun = "==1.2.2" +idna = "==3.4" +importlib-metadata = "==4.13.0" +importlib-resources = "==5.10.1" +iniconfig = "==1.1.1" +jaraco-classes = "==3.2.3" +jeepney = "==0.8.0" +jmespath = "==1.0.1" +jsonpatch = "==1.32" +jsonpointer = "==2.3" +jsonschema = "==4.17.3" +keyring = "==23.11.0" +mccabe = "==0.6.1" +mock = "==4.0.3" +more-itertools = "==9.0.0" +multidict = "==6.0.3" +pkginfo = "==1.9.2" +pkgutil-resolve-name = "==1.3.10" +placebo = "==0.9.0" +pluggy = "==1.0.0" +portalocker = "==2.6.0" +psutil = "==5.9.4" +pycodestyle = "==2.7.0" +pycparser = "==2.21" +pyflakes = "==2.3.1" +pygments = "==2.13.0" +pyrsistent = "==0.19.2" +pytest-cov = "==3.0.0" +pytest-recording = "==0.12.1" +pytest-sugar = "==0.9.6" +pytest-terraform = "==0.6.4" +pytest-xdist = "==3.1.0" +pytest = "==7.2.0" +python-dateutil = "==2.8.2" +pywin32-ctypes = "==0.2.0" +pywin32 = "==305" +pyyaml = "==6.0" +readme-renderer = "==37.3" +requests-toolbelt = "==0.10.1" +requests = "==2.27.1" +rfc3986 = "==2.0.0" +s3transfer = "==0.6.0" +secretstorage = "==3.3.3" +six = "==1.16.0" +tabulate = "==0.8.10" +termcolor = "==2.1.1" +tomli = "==2.0.1" +tqdm = "==4.64.1" +twine = "==3.8.0" +typing-extensions = "==4.4.0" +urllib3 = "==1.26.13" +vcrpy = "==4.2.1" +webencodings = "==0.5.1" +wrapt = "==1.14.1" +yarl = "==1.8.2" +zipp = "==3.11.0" + +[dev-packages] + +[requires] +python_version = "3.11" diff --git a/c7n/manager.py b/c7n/manager.py index d336a3027e5..2f292c2d0d3 100644 --- a/c7n/manager.py +++ b/c7n/manager.py @@ -16,6 +16,7 @@ resources = PluginRegistry('resources') from c7n.utils import dumps +from c7n.resource_metadata_update_with_email import call_api_and_update_resources def iter_filters(filters, block_end=False): @@ -119,7 +120,7 @@ def filter_resources(self, resources, event=None): # NOTE annotate resource ID property. moving this to query.py doesn't work. for r in resources: - if type(r) == dict and "c7n_resource_type_id" not in r: + if isinstance(r, dict) and "c7n_resource_type_id" not in r: try: r["c7n_resource_type_id"] = self.get_model().id except Exception as e: @@ -127,7 +128,18 @@ def filter_resources(self, resources, event=None): self.log.debug("Filtered from %d to %d %s" % ( original, len(resources), self.__class__.__name__.lower())) - return resources + + if not resources or len(resources) == 0: + # If resources is null or empty array, return resources as it is + return resources + else: + try: + updated_resources = call_api_and_update_resources(self, resources) + return updated_resources + except ValueError as error: + print(f"The resources will be returned without modifying the resource metadata for owner emails, as an error occurred: {error}") + # Return the original resources when an error occurs + return resources def get_model(self): """Returns the resource meta-model. diff --git a/c7n/resource_metadata_update_with_email.py b/c7n/resource_metadata_update_with_email.py new file mode 100644 index 00000000000..bb6fcf77a15 --- /dev/null +++ b/c7n/resource_metadata_update_with_email.py @@ -0,0 +1,102 @@ +# This module calls the API gateway, extracts email addresses based on resource tags, account, account's cost center information, and updates resource details with the relevant email data to enhance communication capabilities. + +import os +import json +import requests +import boto3 +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest + +PROVIDERS = { + "AWS": 0, + "Azure": 1, + "GCP": 2, +} + +def extract_appids(resource_list): + appids = {tag.get("Value") for resource in resource_list for tag in resource.get("Tags", []) if tag.get("Key") == "appid"} + return {"appid": list(appids)} + +def call_api_and_update_resources(self, resources, event=None): + try: + appids_data = extract_appids(resources) + # If the "appid" tag is missing , then skip the API call + # Used to avoid api calls for non-DJ accounts + if appids_data != []: + try: + # endpoint = os.environ.get('api_endpoint') + endpoint = 'https://ownerlookupapi.services.dowjones.io' + if not endpoint: + raise ValueError("API endpoint not defined in environment variables.") + + resource_path = '/service' + region = 'us-east-1' + service = 'execute-api' + + session = boto3.Session(region_name=region) + credentials = session.get_credentials() + + appids_data = extract_appids(resources) + if self.account_id: + appids_data["account"] = [self.account_id] + + request = AWSRequest(method='POST', url=endpoint + resource_path, headers={'Content-Type': 'application/json'}) + request.data = json.dumps(appids_data) + SigV4Auth(credentials, service, region).add_auth(request) + + response = requests.post( + request.url, + headers=request.headers, + data=request.data + ) + response.raise_for_status() + response_data = response.json() + + email_address = {} + def extract_from_dict(data_dict, parent_key=""): + nonlocal email_address + for key, value in data_dict.items(): + current_key = f"{parent_key}.{key}" if parent_key else key + if isinstance(value, dict): + extract_from_dict(value, parent_key=current_key) + elif isinstance(value, str) and "@" in value: + if parent_key not in email_address: + email_address[parent_key] = {} + email_address[parent_key][key] = value + + extract_from_dict(response_data) + + for resource in resources: + tags = resource.get("Tags", []) + app_id = next((tag.get("Value") for tag in tags if tag.get("Key") == "appid"), None) + owner_id = self.account_id + + if app_id: + app_email_data = email_address.get(f"appid.{app_id}", {}) + for key, value in app_email_data.items(): + if isinstance(value, str) and "@" in value: + resource[key] = value + + if owner_id: + email_data = email_address.get(f"account.{owner_id}", {}) + for key, value in email_data.items(): + if isinstance(value, str) and "@" in value: + resource[key] = value + + cost_cc_email_data = email_address.get(f"account.{owner_id}.cost_center_info", {}) + for key, value in cost_cc_email_data.items(): + if isinstance(value, str) and "@" in value: + resource[key] = value + + return resources + + except requests.exceptions.RequestException as req_error: + raise ValueError(f"Error making API request: {req_error}") + except (ValueError, json.JSONDecodeError) as json_error: + raise ValueError(f"Error processing API response: {json_error}") + except Exception as error: + raise ValueError(f"Unexpected error occurred: {error}") + except: + resources = resources + return resources + pass diff --git a/c7n/resources/asg.py b/c7n/resources/asg.py index 5cadcc4d6ba..019aca3f210 100644 --- a/c7n/resources/asg.py +++ b/c7n/resources/asg.py @@ -994,7 +994,8 @@ def process(self, asgs): # unless we were given a new value for min_size then # ensure it is at least as low as current_size update['MinSize'] = min(current_size, a['MinSize']) - elif type(self.data['desired-size']) == int: + elif isinstance(self.data['desired-size'], int): + update['DesiredCapacity'] = self.data['desired-size'] if update: diff --git a/c7n/resources/rds.py b/c7n/resources/rds.py index d6c4da49f34..dedc88ac5f2 100644 --- a/c7n/resources/rds.py +++ b/c7n/resources/rds.py @@ -1858,8 +1858,7 @@ def process(self, resources): if r.get( u['property'], jmespath.search( - self.conversion_map.get(u['property'], 'None'), r)) - != u['value']} + self.conversion_map.get(u['property'], 'None'), r)) != u['value']} if not param: continue param['ApplyImmediately'] = self.data.get('immediate', False) diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py index aabb7d12319..39c550ac165 100644 --- a/c7n/resources/vpc.py +++ b/c7n/resources/vpc.py @@ -2885,63 +2885,3 @@ def process(self, resources, event=None): results.append(resource) return results - - -@Subnet.filter_registry.register('ip-allocation-threshold') -class SubnetIpAllocationFilter(Filter): - """Filters subnets based on ip allocation percentage - - :example: - - .. code-block:: yaml - - policies: - - name: subnet-ip-threshold-policy - resource: subnet - filters: - - type: ip-allocation-threshold - percentage: 80 - op: gte - """ - schema = type_schema( - 'ip-allocation-threshold', - percentage={'type': 'number'}, - op={'enum': ['eq', 'ne', 'lt', 'gt', 'lte', 'gte']} - ) - - permissions = ("ec2:DescribeSubnets",) - - def calculate_ip_allocation(self, subnet): - subnetMask = subnet.get('CidrBlock').split('/')[1] - hostBits = 32 - int(subnetMask) - totalHost = ((2 ** hostBits) - 2) - availableHost = subnet.get('AvailableIpAddressCount') - ipsUsed = totalHost - availableHost - percentageOfIpsUsed = (ipsUsed / totalHost) * 100 - return percentageOfIpsUsed - - def process(self, resources, event=None): - results = [] - threshold_percentage = self.data.get('percentage') - op = self.data.get('op') - for subnet in resources: - percentage_used = self.calculate_ip_allocation(subnet) - if op == 'eq': - if threshold_percentage == percentage_used: - results.append(subnet) - elif op == 'ne': - if threshold_percentage != percentage_used: - results.append(subnet) - elif op == 'lt': - if percentage_used < threshold_percentage: - results.append(subnet) - elif op == 'gt': - if percentage_used > threshold_percentage: - results.append(subnet) - elif op == 'lte': - if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): - results.append(subnet) - elif op == 'gte': - if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): - results.append(subnet) - return results diff --git a/tests/conftest.py b/tests/conftest.py index 3c14a12d58d..3d4dd7f8d59 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,7 @@ class LazyPluginCacheDir: pass -pytest_plugins = ("pytest_recording",) +# pytest_plugins = ("pytest_recording",) # If we have C7N_FUNCTIONAL make sure Replay is False otherwise enable Replay LazyReplay.value = not strtobool(os.environ.get('C7N_FUNCTIONAL', 'no')) From 2341eb59b3a0a69e671b4603ff315cbe7316958c Mon Sep 17 00:00:00 2001 From: Gianncarlo Giannattasio Date: Tue, 12 Mar 2024 16:44:32 -0400 Subject: [PATCH 4/6] Added folder option to GCP --- tools/c7n_org/scripts/gcpprojects.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tools/c7n_org/scripts/gcpprojects.py b/tools/c7n_org/scripts/gcpprojects.py index 4a2463aeabb..0e27f6ec898 100644 --- a/tools/c7n_org/scripts/gcpprojects.py +++ b/tools/c7n_org/scripts/gcpprojects.py @@ -13,22 +13,38 @@ help="File to store the generated config (default stdout)") @click.option('-i', '--ignore', multiple=True, help="list of folders that won't be added to the config file") -def main(output, ignore): +@click.option('-fold', '--folders', required=False, multiple=True, + help="List folders the will be added to the config file") +@click.option('-ap','--appscript', default=False, is_flag=True, + help="list of app script projects to account files") +def main(output, ignore, appscript, folders): """ Generate a c7n-org gcp projects config file """ - client = Session().client('cloudresourcemanager', 'v1', 'projects') results = [] for page in client.execute_paged_query('list', {}): for project in page.get('projects', []): + # Exclude App Script GCP Projects + if appscript == False: + if 'sys-' in project['projectId']: + continue + if project['lifecycleState'] != 'ACTIVE': continue if project["parent"]["id"] in ignore: continue + + if folders != (): + if project["parent"]["type"] != "folder": + continue + for fold in folders: + if project["parent"]["id"] != fold: + continue + project_info = { 'project_id': project['projectId'], @@ -47,4 +63,4 @@ def main(output, ignore): if __name__ == '__main__': - main() + main() \ No newline at end of file From 4d627490cfcaa889866fb191a79db2e842d05d81 Mon Sep 17 00:00:00 2001 From: Gianncarlo Giannattasio Date: Tue, 12 Mar 2024 16:59:40 -0400 Subject: [PATCH 5/6] readd ip-allocation-threshold --- c7n/resources/vpc.py | 56 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py index 39c550ac165..d122f807ccc 100644 --- a/c7n/resources/vpc.py +++ b/c7n/resources/vpc.py @@ -2885,3 +2885,59 @@ def process(self, resources, event=None): results.append(resource) return results + +@Subnet.filter_registry.register('ip-allocation-threshold') +class SubnetIpAllocationFilter(Filter): + """Filters subnets based on ip allocation percentage + :example: + .. code-block:: yaml + policies: + - name: subnet-ip-threshold-policy + resource: subnet + filters: + - type: ip-allocation-threshold + percentage: 80 + op: gte + """ + schema = type_schema( + 'ip-allocation-threshold', + percentage={'type': 'number'}, + op={'enum': ['eq', 'ne', 'lt', 'gt', 'lte', 'gte']} + ) + + permissions = ("ec2:DescribeSubnets",) + + def calculate_ip_allocation(self, subnet): + subnetMask = subnet.get('CidrBlock').split('/')[1] + hostBits = 32 - int(subnetMask) + totalHost = ((2 ** hostBits) - 2) + availableHost = subnet.get('AvailableIpAddressCount') + ipsUsed = totalHost - availableHost + percentageOfIpsUsed = (ipsUsed / totalHost) * 100 + return percentageOfIpsUsed + + def process(self, resources, event=None): + results = [] + threshold_percentage = self.data.get('percentage') + op = self.data.get('op') + for subnet in resources: + percentage_used = self.calculate_ip_allocation(subnet) + if op == 'eq': + if threshold_percentage == percentage_used: + results.append(subnet) + elif op == 'ne': + if threshold_percentage != percentage_used: + results.append(subnet) + elif op == 'lt': + if percentage_used < threshold_percentage: + results.append(subnet) + elif op == 'gt': + if percentage_used > threshold_percentage: + results.append(subnet) + elif op == 'lte': + if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + elif op == 'gte': + if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + return results \ No newline at end of file From 8fb18592d32d236238966495b65cb378a24f4dff Mon Sep 17 00:00:00 2001 From: Wayne Cierkowski Date: Mon, 22 Apr 2024 12:37:40 -0400 Subject: [PATCH 6/6] IE-3315: Added filter to project query --- tools/c7n_org/scripts/gcpprojects.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/c7n_org/scripts/gcpprojects.py b/tools/c7n_org/scripts/gcpprojects.py index 4baefa453b1..ab50ee7bdbc 100644 --- a/tools/c7n_org/scripts/gcpprojects.py +++ b/tools/c7n_org/scripts/gcpprojects.py @@ -23,14 +23,12 @@ def main(output, exclude, buid, appscript): """ client = Session().client('cloudresourcemanager', 'v1', 'projects') + query_params = {'filter': f"parent.type:folder parent.id:{buid}"} if buid else {} results = [] - for page in client.execute_paged_query('list', {}): + for page in client.execute_paged_query('list', query_params): for project in page.get('projects', []): - if buid and project["parent"]["id"] != buid: - continue - # Exclude App Script GCP Projects if appscript == False: if 'sys-' in project['projectId']: