Merge branch 'upstream-gcp-mailer' into develop

newscorp-ghfb · Aug 31, 2022 · 2b23cf8 · 2b23cf8
2 parents 41d03be + b1fd240
commit 2b23cf8
Show file tree

Hide file tree

Showing 33 changed files with 2,724 additions and 1,779 deletions.
diff --git a/poetry.lock b/poetry.lock
diff --git a/tools/c7n_mailer/README.md b/tools/c7n_mailer/README.md
@@ -84,7 +84,7 @@ The standard way to do a DataDog integration is use the
 c7n integration with AWS CloudWatch and use the
 [DataDog integration with AWS](https://docs.datadoghq.com/integrations/amazon_web_services/)
 to collect CloudWatch metrics. The mailer/messenger integration is only
-for the case you don't want or you can't use AWS CloudWatch.
+for the case you don't want or you can't use AWS CloudWatch, e.g. in Azure or GCP.
 
 Note this integration requires the additional dependency of datadog python bindings:
 ```
@@ -266,12 +266,14 @@ configuration you specify in a YAML file.  Here is [the
 schema](./c7n_mailer/cli.py#L11-L41) to which the file must conform,
 and here is a description of the options:
 
-| Required? | Key             | Type             | Notes                                                                                                                                                                               |
-|:---------:|:----------------|:-----------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| &#x2705;  | `queue_url`     | string           | the queue to listen to for messages                                                                                                                                                 |
-|           | `from_address`  | string           | default from address                                                                                                                                                                |
-|           | `endpoint_url`  | string           | SQS API URL (for use with VPC Endpoints)                                                                                                                                                                |
-|           | `contact_tags`  | array of strings | tags that we should look at for address information                                                                                                                                 |
+| Required? | Key             | Type             | Notes                                                             |
+|:---------:|:----------------|:-----------------|:------------------------------------------------------------------|
+| &#x2705;  | `queue_url`     | string           | the queue to listen to for messages                               | 
+|           | `from_address`  | string           | default from address                                              |
+|           | `endpoint_url`  | string           | SQS API URL (for use with VPC Endpoints)                          |
+|           | `contact_tags`  | array of strings | tags that we should look at for address information               |
+|           | `email_base_url`| string           | Base URL to construct a valid email address from a resource owner |
+
 
 ### Standard Lambda Function Config
 
@@ -291,9 +293,9 @@ and here is a description of the options:
 | Required? | Key                   | Type   | Notes                                                                                  |
 |:---------:|:----------------------|:-------|:---------------------------------------------------------------------------------------|
 |           | `function_properties` | object | Contains `appInsights`, `storageAccount` and `servicePlan` objects                     |
-|           | `appInsights`         | object | Contains `name`, `location` and `resourceGroupName` properties                       |
-|           | `storageAccount`      | object | Contains `name`, `location` and `resourceGroupName` properties                       |
-|           | `servicePlan`         | object | Contains `name`, `location`, `resourceGroupName`, `skuTier` and `skuName` properties |
+|           | `appInsights`         | object | Contains `name`, `location` and `resourceGroupName` properties                         |
+|           | `storageAccount`      | object | Contains `name`, `location` and `resourceGroupName` properties                         |
+|           | `servicePlan`         | object | Contains `name`, `location`, `resourceGroupName`, `skuTier` and `skuName` properties   |
 |           | `name`                | string |                                                                                        |
 |           | `location`            | string | Default: `west us 2`                                                                   |
 |           | `resourceGroupName`   | string | Default `cloud-custodian`                                                              |
@@ -312,7 +314,7 @@ and here is a description of the options:
 |           | `debug`                     | boolean | debug on/off                                                                                                                                                                                       |
 |           | `ldap_bind_dn`              | string  | eg: ou=people,dc=example,dc=com                                                                                                                                                                    |
 |           | `ldap_bind_user`            | string  | eg: FOO\\BAR                                                                                                                                                                                       |
-|           | `ldap_bind_password`        | string  | ldap bind password                                                                                                                                                                                 |
+|           | `ldap_bind_password`        | secured string  | ldap bind password                                                                                                                                                                                 |
 |           | `ldap_bind_password_in_kms` | boolean | defaults to true, most people (except capone) want to set this to false. If set to true, make sure `ldap_bind_password` contains your KMS encrypted ldap bind password as a base64-encoded string. |
 |           | `ldap_email_attribute`      | string  |                                                                                                                                                                                                    |
 |           | `ldap_email_key`            | string  | eg 'mail'                                                                                                                                                                                          |
@@ -410,6 +412,27 @@ You can store your secrets in Azure Key Vault secrets and reference them from th
 
 Note: `secrets.get` permission on the KeyVault for the Service Principal is required.
 
+#### GCP
+
+You can store your secrets as GCP Secret Manager secrets and reference them from the policy.
+
+```yaml
+  plaintext_secret: <raw_secret>
+  secured_string:
+    type: gcp.secretmanager
+    secret: projects/12345678912/secrets/your-secret
+```
+
+An example of an SMTP password set as a secured string:
+
+```yaml
+  smtp_password:
+    type: gcp.secretmanager
+    secret: projects/59808015552/secrets/smtp_pw
+```
+
+Note: If you do not specify a version, `/versions/latest` will be appended to your secret location.
+
 ## Configuring a policy to send email
 
 Outbound email can be added to any policy by including the `notify` action.
@@ -601,6 +624,64 @@ function_properties:
     type: SystemAssigned
 ```
 
+## Using on GCP
+
+Requires:
+
+- `c7n_gcp` package.  See [GCP Getting Started](https://cloudcustodian.io/docs/gcp/gettingstarted.html)
+- `google-cloud-secret-manager` package, for pulling in secured string values.
+- A working SMTP Account.
+- [GCP Pubsub Subscription](https://cloud.google.com/pubsub/docs/)
+
+The mailer supports GCP Pubsub transports and SMTP/Email delivery, as well as Datadog and Splunk.
+Configuration for this scenario requires only minor changes from AWS deployments.
+
+The notify action in your policy will reflect transport type `projects` with the URL
+to a GCP Pub/Sub Topic.  For example:
+
+```yaml
+policies:
+  - name: gcp-notify
+    resource: gcp.compute
+    description: example policy
+    actions:
+      - type: notify
+        template: default
+        priority_header: '2'
+        subject: Hello from C7N Mailer
+        to:
+          - [email protected]
+        transport:
+          type: pubsub
+          topic: projects/myproject/topics/mytopic
+```
+
+In your mailer configuration, you'll need to provide your SMTP account information
+as well as your topic subscription path in the queue_url variable. Please note that the
+subscription you specify should be subscribed to the topic you assign in your policies'
+notify action for GCP resources.
+
+```yaml
+queue_url: projects/myproject/subscriptions/mysubscription
+from_address: [email protected]
+# c7n-mailer currently requires a role be present, even if it's empty
+role: ""
+
+smtp_server: my.smtp.add.ress
+smtp_port: 25
+smtp_ssl: true
+smtp_username: smtpuser
+smtp_password:
+  type: gcp.secretmanager
+  secret: projects/12345678912/secrets/smtppassword
+```
+
+The mailer will transmit all messages found on the queue on each execution using SMTP/Email delivery.
+
+### Deploying GCP Functions
+
+GCP Cloud Functions for c7n-mailer are currently not supported.
+
 ## Writing an email template
 
 Templates are authored in [jinja2](http://jinja.pocoo.org/docs/dev/templates/).

diff --git a/tools/c7n_mailer/c7n_mailer/azure_mailer/azure_queue_processor.py b/tools/c7n_mailer/c7n_mailer/azure_mailer/azure_queue_processor.py
@@ -11,6 +11,7 @@
 
 from c7n_mailer.azure_mailer.sendgrid_delivery import SendGridDelivery
 from c7n_mailer.smtp_delivery import SmtpDelivery
+from c7n_mailer.target import MessageTargetMixin
 
 try:
     from c7n_azure.storage_utils import StorageUtilities
@@ -21,7 +22,7 @@
     pass
 
 
-class MailerAzureQueueProcessor:
+class MailerAzureQueueProcessor(MessageTargetMixin):
 
     def __init__(self, config, logger, session=None, max_num_processes=16):
         if StorageUtilities is None:
@@ -48,7 +49,9 @@ def run(self, parallel=False):
             for queue_message in queue_messages:
                 self.logger.debug("Message id: %s received" % queue_message.id)
 
-                if (self.process_azure_queue_message(queue_message) or
+                if (
+                    self.process_azure_queue_message(
+                        queue_message, str(queue_message.inserted_on)) or
                         queue_message.dequeue_count > self.max_message_retry):
                     # If message handled successfully or max retry hit, delete
                     StorageUtilities.delete_queue_message(*queue_settings, message=queue_message)
@@ -58,7 +61,7 @@ def run(self, parallel=False):
 
         self.logger.info('No messages left on the azure storage queue, exiting c7n_mailer.')
 
-    def process_azure_queue_message(self, encoded_azure_queue_message):
+    def process_azure_queue_message(self, encoded_azure_queue_message, timestamp):
         queue_message = json.loads(
             zlib.decompress(base64.b64decode(encoded_azure_queue_message.content)))
 
@@ -70,12 +73,7 @@ def process_azure_queue_message(self, encoded_azure_queue_message):
             queue_message['policy']['name'],
             ', '.join(queue_message['action'].get('to', []))))
 
-        if any(e.startswith('slack') or e.startswith('https://hooks.slack.com/')
-                for e in queue_message.get('action', ()).get('to', [])):
-            self._deliver_slack_message(queue_message)
-
-        if any(e.startswith('datadog') for e in queue_message.get('action', ()).get('to', [])):
-            self._deliver_datadog_message(queue_message)
+        self.handle_targets(queue_message, timestamp)
 
         email_result = self._deliver_email(queue_message)
 
@@ -84,29 +82,6 @@ def process_azure_queue_message(self, encoded_azure_queue_message):
         else:
             return True
 
-    def _deliver_slack_message(self, queue_message):
-        from c7n_mailer.slack_delivery import SlackDelivery
-        slack_delivery = SlackDelivery(self.config,
-                                       self.logger,
-                                       SendGridDelivery(self.config, self.session, self.logger))
-        slack_messages = slack_delivery.get_to_addrs_slack_messages_map(queue_message)
-        try:
-            self.logger.info('Sending message to Slack.')
-            slack_delivery.slack_handler(queue_message, slack_messages)
-        except Exception as error:
-            self.logger.exception(error)
-
-    def _deliver_datadog_message(self, queue_message):
-        from c7n_mailer.datadog_delivery import DataDogDelivery
-        datadog_delivery = DataDogDelivery(self.config, self.session, self.logger)
-        datadog_message_packages = datadog_delivery.get_datadog_message_packages(queue_message)
-
-        try:
-            self.logger.info('Sending message to Datadog.')
-            datadog_delivery.deliver_datadog_messages(datadog_message_packages, queue_message)
-        except Exception as error:
-            self.logger.exception(error)
-
     def _deliver_email(self, queue_message):
         try:
             sendgrid_delivery = SendGridDelivery(self.config, self.session, self.logger)

diff --git a/tools/c7n_mailer/c7n_mailer/cli.py b/tools/c7n_mailer/c7n_mailer/cli.py
@@ -9,9 +9,11 @@
 import jsonschema
 import yaml
 from c7n_mailer import deploy, utils
+from c7n_mailer.sqs_queue_processor import MailerSqsQueueProcessor
 from c7n_mailer.azure_mailer.azure_queue_processor import MailerAzureQueueProcessor
+from c7n_mailer.gcp_mailer.gcp_queue_processor import MailerGcpQueueProcessor
 from c7n_mailer.azure_mailer import deploy as azure_deploy
-from c7n_mailer.sqs_queue_processor import MailerSqsQueueProcessor
+# from c7n_mailer.gcp_mailer import deploy as gcp_deploy
 from c7n_mailer.utils import get_provider, Providers
 
 AZURE_KV_SECRET_SCHEMA = {
@@ -24,10 +26,21 @@
     'additionalProperties': False
 }
 
+GCP_SECRET_SCHEMA = {
+    'type': 'object',
+    'properties': {
+        'type': {'enum': ['gcp.secretmanager']},
+        'secret': {'type': 'string'}
+    },
+    'required': ['type', 'secret'],
+    'additionalProperties': False
+}
+
 SECURED_STRING_SCHEMA = {
     'oneOf': [
         {'type': 'string'},
-        AZURE_KV_SECRET_SCHEMA
+        AZURE_KV_SECRET_SCHEMA,
+        GCP_SECRET_SCHEMA
     ]
 }
 
@@ -38,6 +51,7 @@
     'additionalProperties': False,
     'required': ['queue_url'],
     'properties': {
+        'cloud_provider': {'type': 'string'},
         'queue_url': {'type': 'string'},
         'gcp_queue_url': {'type': 'string'},
         'service_account_info': {'type': 'string'},
@@ -118,9 +132,11 @@
                 ]
             },
         },
-        'function_schedule': {'type': 'string'},
-        'function_skuCode': {'type': 'string'},
-        'function_sku': {'type': 'string'},
+        # GCP Cloud Function Config # TODO:
+        # 'function_schedule': {'type': 'string'},
+        # 'function_skuCode': {'type': 'string'},
+        # 'function_sku': {'type': 'string'},
+        'email_base_url': {'type': 'string'},
 
         # Mailer Infrastructure Config
         'cache_engine': {'type': 'string'},
@@ -140,7 +156,7 @@
         'ldap_manager_attribute': {'type': 'string'},
         'ldap_email_attribute': {'type': 'string'},
         'ldap_bind_password_in_kms': {'type': 'boolean'},
-        'ldap_bind_password': {'type': 'string'},
+        'ldap_bind_password': SECURED_STRING_SCHEMA,
         'cross_accounts': {'type': 'object'},
         'ses_region': {'type': 'string'},
         'redis_host': {'type': 'string'},
@@ -265,6 +281,8 @@ def main():
 
         if provider == Providers.Azure:
             azure_deploy.provision(mailer_config)
+        # elif provider == Providers.GCP:  # TODO:
+        #     gcp_deploy.provision(mailer_config)
         elif provider == Providers.AWS:
             deploy.provision(mailer_config, functools.partial(session_factory, mailer_config))
 
@@ -274,6 +292,8 @@ def main():
         # Select correct processor
         if provider == Providers.Azure:
             processor = MailerAzureQueueProcessor(mailer_config, logger)
+        elif provider == Providers.GCP:
+            processor = MailerGcpQueueProcessor(mailer_config, logger)
         elif provider == Providers.AWS:
             aws_session = session_factory(mailer_config)
             processor = MailerSqsQueueProcessor(mailer_config, aws_session, logger)

diff --git a/tools/c7n_mailer/c7n_mailer/email_delivery.py b/tools/c7n_mailer/c7n_mailer/email_delivery.py
@@ -4,13 +4,13 @@
 from itertools import chain
 
 from c7n_mailer.smtp_delivery import SmtpDelivery
-from c7n_mailer.utils_email import is_email, get_mimetext_message
 import c7n_mailer.azure_mailer.sendgrid_delivery as sendgrid
 
 from .ldap_lookup import LdapLookup
 from .utils import (
-    get_resource_tag_targets,
-    kms_decrypt, get_aws_username_from_event)
+    decrypt, get_resource_tag_targets, get_provider,
+    get_aws_username_from_event, Providers)
+from .utils_email import get_mimetext_message, is_email
 
 
 class EmailDelivery:
@@ -19,28 +19,40 @@ def __init__(self, config, session, logger):
         self.config = config
         self.logger = logger
         self.session = session
-        self.aws_ses = session.client('ses', region_name=config.get('ses_region'))
+        self.provider = get_provider(self.config)
+        if self.provider == Providers.AWS:
+            self.aws_ses = session.client('ses', region_name=config.get('ses_region'))
         self.ldap_lookup = self.get_ldap_connection()
         self.servicenow_url = config.get("servicenow_url")
 
     def get_ldap_connection(self):
         if self.config.get('ldap_uri'):
-            self.config['ldap_bind_password'] = kms_decrypt(self.config, self.logger,
-                                                            self.session, 'ldap_bind_password')
+            credential = decrypt(
+                self.config, self.logger, self.session, 'ldap_bind_password')
+            self.config['ldap_bind_password'] = credential
             return LdapLookup(self.config, self.logger)
         return None
 
     def get_valid_emails_from_list(self, targets):
         emails = []
         for target in targets:
+            if target in ('resource-owner', 'event-owner'):
+                continue
             target = target.replace(";", ",")
             for email in target.split(','):
                 email = email.strip()
                 if is_email(email):
                     emails.append(email)
+                # gcp doesn't support the '@' character in their label values so we
+                # allow users to specify an email_base_url to append to the end of their
+                # owner contact tags
+                if not is_email(target) and self.config.get('email_base_url'):
+                    target = "%s@%s" % (target, self.config['email_base_url'])
+                    if is_email(target):
+                        emails.append(target)
         return emails
 
-    def get_event_owner_email(self, targets, event):
+    def get_event_owner_email(self, targets, event):  # TODO: GCP-friendly
         if 'event-owner' in targets:
             aws_username = get_aws_username_from_event(self.logger, event)
             if aws_username:
@@ -112,7 +124,7 @@ def get_resource_owner_emails_from_resource(self, sqs_message, resource):
 
         return list(chain(explicit_emails, ldap_emails, org_emails))
 
-    def get_account_emails(self, sqs_message):
+    def get_account_emails(self, sqs_message):  # TODO: GCP-friendly
         email_list = []
 
         if 'account-emails' not in sqs_message['action'].get('to', []):
@@ -261,6 +273,7 @@ def get_group_email_messages_map(self, sqs_message):
         return groupby_to_mimetext_map
 
     def send_c7n_email(self, sqs_message, email_to_addrs, mimetext_msg):
+        print("send_c7n_email", self.config.get("sendgrid_api_key"))
         try:
             # if smtp_server is set in mailer.yml, send through smtp
             if 'smtp_server' in self.config:

diff --git a/tools/c7n_mailer/c7n_mailer/gcp_mailer/__init__.py b/tools/c7n_mailer/c7n_mailer/gcp_mailer/__init__.py
@@ -0,0 +1,2 @@
+# Copyright The Cloud Custodian Authors.
+# SPDX-License-Identifier: Apache-2.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# Copyright The Cloud Custodian Authors.
		# SPDX-License-Identifier: Apache-2.0