lambdas now take parameters

Author: mvanholsteijn

README.md

@@ -1,10 +1,39 @@
 # AWS Cloudwatch Log minder
-the AWS Cloudwatch Log minder, has two tasks:
-- remove empty log streams older than the retention period of the log group
-- set a default retention period on log groups without a period set.
+AWS CloudWatch logs is an useful logging system, but it has two quircks. It does not allow you too set a default
+retention period for newly created log groups, and it does not delete empty log streams that are older than
+the retention period. This utility provides two functions:
+
+1. set a default retention period on log groups without a period set.
+1. remove empty log streams older than the retention period of the log group
+
+You can use it as a command line utility, or better, install it as an AWS Lambda function and have your
+log kept in order every day, NoOps style!
+
+## install the log minder
+to install the log minder, type:
+
+```sh
+pip install aws-cloudwatch-log-minder
+```
+
+## set default retention period
+to set the default retention period on log groups without one, type:
+```sh
+cwlogs-minder --dry-run set-log-retention --days 30
+```
+This will show you which log groups, will have its retention period set. Remove the `--dry-run` and
+it will actually be done.
+
+## delete empty log streams
+To delete empty log streams older than the retention period, type:
+```sh
+cwlogs-minder --dry-run delete-empty-log-streams
+```
+This will show you which empty log streams will be deleted. Remove the `--dry-run` and
+they actually will.
 
 ## deploy the log minder
-To deploy the log minder, type:
+To deploy the log minder as an AWS Lambda, type:
 
 ```sh
 git clone https://github.com/binxio/aws-cloudwatch-log-minder.git
@@ -16,11 +45,5 @@ aws cloudformation create-stack \
 
 aws cloudformation wait stack-create-complete  --stack-name aws-cloudwatch-log-minder
 ```
-
-It is scheduled to run every day around midnight.
-
-You can also use the command line utility:
-
-```sh
-pip install aws-cloudwatch-log-minder
-```
+This will install the log minder in your AWS account and run around midnight. First setting 
+retention periods and delete the empty log streams secondly.

cloudformation/aws-cloudwatch-log-minder.yaml

@@ -25,7 +25,7 @@ Resources:
           Action:
             - logs:DescribeLogGroups
             - logs:DescribeLogStreams
-            - log:PutRetentionPolicy
+            - logs:PutRetentionPolicy
             - logs:DeleteLogStreams
           Resource: "*"
 

src/aws_cloudwatch_log_minder/delete_empty_log_streams.py

@@ -4,45 +4,66 @@
 from .logging import log
 from datetime import datetime
 
-cw_logs = boto3.client('logs')
+cw_logs = boto3.client("logs")
 
 
 def _delete_empty_log_streams(group: dict, oldest_in_ms: int, dry_run: bool):
     log_group_name = group["logGroupName"]
     kwargs = {
         "logGroupName": log_group_name,
-        "orderBy": 'LastEventTime',
-        "descending": False
+        "orderBy": "LastEventTime",
+        "descending": False,
     }
-    for response in cw_logs.get_paginator('describe_log_streams').paginate(**kwargs):
-        for stream in response['logStreams']:
-            log_stream_name = stream['logStreamName']
+    for response in cw_logs.get_paginator("describe_log_streams").paginate(**kwargs):
+        for stream in response["logStreams"]:
+            log_stream_name = stream["logStreamName"]
             if stream["creationTime"] > oldest_in_ms:
-                log.debug('keeping log stream %s from group %s as it is within retention period', log_stream_name, log_group_name)
+                log.debug(
+                    "keeping log stream %s from group %s as it is within retention period",
+                    log_stream_name,
+                    log_group_name,
+                )
                 continue
 
             if not stream["storedBytes"] == 0 and stream["creationTime"] < oldest_in_ms:
                 try:
-                    log.info('deleting log stream %s from group %s', log_stream_name, log_group_name)
+                    log.info(
+                        "deleting log stream %s from group %s",
+                        log_stream_name,
+                        log_group_name,
+                    )
                     if dry_run:
                         continue
                     cw_logs.delete_log_stream(
-                        logGroupName=log_group_name,
-                        logStreamName=log_stream_name
+                        logGroupName=log_group_name, logStreamName=log_stream_name
                     )
                 except ClientError as e:
-                    log.error('failed to delete log stream %s from group %s, %s', log_stream_name, log_group_name, e)
+                    log.error(
+                        "failed to delete log stream %s from group %s, %s",
+                        log_stream_name,
+                        log_group_name,
+                        e,
+                    )
             else:
-                log.debug('keeping log stream %s from group %s', log_stream_name, log_group_name)
+                log.debug(
+                    "keeping log stream %s from group %s",
+                    log_stream_name,
+                    log_group_name,
+                )
+
 
 def delete_empty_log_streams(dry_run: bool = False):
-    log.info('cleaning empty log streams older than the retention period of the group')
+    log.info("cleaning empty log streams older than the retention period of the group")
     now = (datetime.utcnow() - datetime(1970, 1, 1)).total_seconds()
-    for response in cw_logs.get_paginator('describe_log_groups').paginate():
-        for group in response['logGroups']:
-            oldest = now - (group.get('retentionInDays', 365) * 24 * 3600)
+    for response in cw_logs.get_paginator("describe_log_groups").paginate():
+        for group in response["logGroups"]:
+            oldest = now - (group.get("retentionInDays", 365) * 24 * 3600)
             _delete_empty_log_streams(group, oldest * 1000, dry_run)
 
 
-def handle(request, context):
-    delete_empty_log_streams()
+def handle(request: dict = {}, context: dict = {}):
+    dry_run = request.get("dry_run", False)
+    if "dry_run" in request and not isinstance(dry_run, bool):
+        raise ValueError(f"'dry_run' is not a boolean value, {request}")
+
+    delete_empty_log_streams(dry_run)

src/aws_cloudwatch_log_minder/set_log_retention.py

@@ -3,31 +3,50 @@
 from botocore.exceptions import ClientError
 from .logging import log
 
-cw_logs = boto3.client('logs')
+cw_logs = boto3.client("logs")
 
-def set_log_retention(default_retention_period:int = 30, dry_run: bool = False):
-    for response in cw_logs.get_paginator('describe_log_groups').paginate():
-        for group in response['logGroups']:
+
+def set_log_retention(retention_in_days: int = 30, dry_run: bool = False):
+    for response in cw_logs.get_paginator("describe_log_groups").paginate():
+        for group in response["logGroups"]:
             log_group_name = group["logGroupName"]
-            current_retention = group.get('retentionInDays')
+            current_retention = group.get("retentionInDays")
             if not current_retention:
                 try:
-                    log.info('setting default retention period of log stream %s to %s', log_group_name, default_retention_period)
+                    log.info(
+                        "setting default retention period of log stream %s to %s",
+                        log_group_name,
+                        retention_in_days,
+                    )
                     if dry_run:
                         continue
                     cw_logs.put_retention_policy(
                         logGroupName=log_group_name,
-                        retentionInDays=default_retention_period
+                        retentionInDays=retention_in_days,
                     )
                 except ClientError as e:
-                    log.error('failed to set retention period of log stream %s to %s, %s', log_group_name, default_retention_period, e)
+                    log.error(
+                        "failed to set retention period of log stream %s to %s, %s",
+                        log_group_name,
+                        retention_in_days,
+                        e,
+                    )
             else:
-                log.debug('retention period on %s already set to %s', log_group_name, current_retention)
+                log.debug(
+                    "retention period on %s already set to %s",
+                    log_group_name,
+                    current_retention,
+                )
 
 
 def handle(request={}, context={}):
-    log.info('setting default retention period on newly created log groups')
-    default_retention_period = int(os.getenv('DEFAULT_RETENTION_PERIOD_IN_DAYS', '30'))
-    set_log_retention(default_retention_period)
+    dry_run = request.get("dry_run", False)
+    if "dry_run" in request and not isinstance(dry_run, bool):
+        raise ValueError(f"'dry_run' is not a boolean value, {request}")
 
+    default_log_retention = int(os.getenv("DEFAULT_LOG_RETENTION_IN_DAYS", "30"))
+    days = request.get("days", default_log_retention)
+    if "days" in request and not isinstance(days, int):
+        raise ValueError(f"'days' is not a integer value, {request}")
 
+    set_log_retention(days, dry_run)