Adding image cleanup for clouds

cloudwash/cli.py

@@ -67,6 +67,7 @@ def gce(ctx, vms, discs, nics, _all):
 
 @cleanup_providers.command(help="Cleanup Azure provider")
 @common_options
+@click.option("--images", is_flag=True, help="Remove only images from the provider")
 @click.option("--pips", is_flag=True, help="Remove only PiPs from the provider")
 @click.option(
     "--all_rg",
@@ -75,14 +76,15 @@ def gce(ctx, vms, discs, nics, _all):
     help="Remove resource group only if all resources are older than SLA",
 )
 @click.pass_context
-def azure(ctx, vms, discs, nics, pips, _all, _all_rg):
+def azure(ctx, vms, discs, nics, images, pips, _all, _all_rg):
     # Validate Azure Settings
     validate_provider(ctx.command.name)
     is_dry_run = ctx.parent.params["dry"]
     azureCleanup(
         vms=vms,
         discs=discs,
         nics=nics,
+        images=images,
         pips=pips,
         _all=_all,
         _all_rg=_all_rg,
@@ -92,15 +94,23 @@ def azure(ctx, vms, discs, nics, pips, _all, _all_rg):
 
 @cleanup_providers.command(help="Cleanup Amazon provider")
 @common_options
+@click.option("--images", is_flag=True, help="Remove only images from the provider")
 @click.option("--pips", is_flag=True, help="Remove only Public IPs from the provider")
 @click.option("--stacks", is_flag=True, help="Remove only CloudFormations from the provider")
 @click.pass_context
-def aws(ctx, vms, discs, nics, pips, stacks, _all):
+def aws(ctx, vms, discs, nics, images, pips, stacks, _all):
     # Validate Amazon Settings
     validate_provider(ctx.command.name)
     is_dry_run = ctx.parent.params["dry"]
     awsCleanup(
-        vms=vms, discs=discs, nics=nics, pips=pips, stacks=stacks, _all=_all, dry_run=is_dry_run
+        vms=vms,
+        discs=discs,
+        nics=nics,
+        images=images,
+        pips=pips,
+        stacks=stacks,
+        _all=_all,
+        dry_run=is_dry_run,
     )
 
 

cloudwash/providers/aws.py

@@ -53,6 +53,27 @@ def dry_discs():
                     [dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs]
                 return rdiscs
 
+            def dry_images():
+                rimages = []
+                if settings.aws.criteria.image.unassigned:
+                    rimages = aws_client.list_templates(executable_by_me=False, owned_by_me=True)
+                    free_images = aws_client.list_free_images(
+                        image_list=[image.raw.image_id for image in rimages]
+                    )
+                    remove_images = [
+                        image
+                        for image in free_images
+                        if image not in settings.aws.exceptions.images
+                    ]
+                    if settings.aws.criteria.image.delete_image:
+                        remove_images = [
+                            image
+                            for image in remove_images
+                            if image.startswith(settings.aws.criteria.image.delete_image)
+                        ]
+                    dry_data["IMAGES"]["delete"].extend(remove_images)
+                return remove_images
+
             def dry_pips():
                 rpips = []
                 if settings.aws.criteria.public_ip.unassigned:
@@ -106,6 +127,12 @@ def remove_stacks(stacks):
                 if not is_dry_run and rdiscs:
                     aws_client.remove_all_unused_volumes()
                     logger.info(f"Removed Discs: \n{rdiscs}")
+            if kwargs["images"] or kwargs["_all"]:
+                rimages = dry_images()
+                if not is_dry_run and rimages:
+                    aws_client.delete_images(image_list=rimages)
+                    logger.info(f"Removed Images: \n{rimages}")
+
             if kwargs["pips"] or kwargs["_all"]:
                 rpips = dry_pips()
                 if not is_dry_run and rpips:

cloudwash/providers/azure.py

@@ -39,7 +39,7 @@ def _dry_vms(all_vms):
 def cleanup(**kwargs):
     is_dry_run = kwargs["dry_run"]
 
-    data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES']
+    data = ['VMS', 'NICS', 'DISCS', 'IMAGES', 'PIPS', 'RESOURCES']
     regions = settings.azure.auth.regions
     groups = settings.azure.auth.resource_groups
 
@@ -97,6 +97,28 @@ def dry_resources(hours_old=None):
                     )
                     return dry_data["RESOURCES"]["delete"]
 
+                def dry_images():
+                    remove_images = []
+                    if settings.azure.criteria.image.unassigned:
+                        images_list = azure_client.list_compute_images_by_resource_group(
+                            free_images=True
+                        )
+                        image_names = [image.name for image in images_list]
+                        # Filter out the images not to be removed.
+                        remove_images = [
+                            image
+                            for image in image_names
+                            if image not in settings.azure.exceptions.images
+                        ]
+                        if settings.azure.criteria.image.delete_image:
+                            remove_images = [
+                                image
+                                for image in remove_images
+                                if image.startswith(settings.azure.criteria.image.delete_image)
+                            ]
+                        dry_data["IMAGES"]["delete"].extend(remove_images)
+                    return remove_images
+
                 # Remove / Stop VMs
                 def remove_vms(avms):
                     # Remove VMs
@@ -129,6 +151,12 @@ def remove_vms(avms):
                     if not is_dry_run and rpips:
                         azure_client.remove_pips_by_search()
                         logger.info(f"Removed PIPs: \n{rpips}")
+                if kwargs["images"] or kwargs["_all"]:
+                    rimages = dry_images()
+                    if not is_dry_run and rimages:
+                        azure_client.delete_compute_image_by_resource_group(image_list=rimages)
+                        logger.info(f"Removed Images: \n{rimages}")
+
                 if kwargs["_all_rg"]:
                     sla_time = settings.azure.criteria.resource_group.resources_sla_minutes
 

cloudwash/utils.py

@@ -13,6 +13,7 @@
     "PIPS": {"delete": []},
     "RESOURCES": {"delete": []},
     "STACKS": {"delete": []},
+    "IMAGES": {"delete": []},
 }
 dry_data.update(_vms_dict)
 
@@ -29,6 +30,7 @@ def echo_dry(dry_data=None) -> None:
     skipped_vms = dry_data["VMS"]["skip"]
     deletable_discs = dry_data["DISCS"]["delete"]
     deletable_nics = dry_data["NICS"]["delete"]
+    deletable_images = dry_data["IMAGES"]["delete"]
     deletable_pips = dry_data["PIPS"]["delete"] if "PIPS" in dry_data else None
     deletable_resources = dry_data["RESOURCES"]["delete"]
     deletable_stacks = dry_data["STACKS"]["delete"] if "STACKS" in dry_data else None
@@ -41,6 +43,8 @@ def echo_dry(dry_data=None) -> None:
         logger.info(f"DISCs:\n\tDeletable: {deletable_discs}")
     if deletable_nics:
         logger.info(f"NICs:\n\tDeletable: {deletable_nics}")
+    if deletable_images:
+        logger.info(f"IMAGES:\n\tDeletable: {deletable_images}")
     if deletable_pips:
         logger.info(f"PIPs:\n\tDeletable: {deletable_pips}")
     if deletable_resources:
@@ -56,6 +60,7 @@ def echo_dry(dry_data=None) -> None:
             deletable_pips,
             deletable_resources,
             deletable_stacks,
+            deletable_images,
         ]
     ):
         logger.info("\nNo resources are eligible for cleanup!")

conf/aws.yaml.template

@@ -13,6 +13,11 @@ AWS:
         DISC:
             UNASSIGNED: True
         NIC:
+            UNASSIGNED: True
+        IMAGE:
+            # Image name starts with
+            DELETE_IMAGE:
+
             UNASSIGNED: True
         PUBLIC_IP:
             UNASSIGNED: True
@@ -30,3 +35,4 @@ AWS:
         STACKS:
             # CloudFormations names that would be skipped from cleanup
             STACK_LIST: []
+        IMAGES: []

conf/azure.yaml.template

@@ -17,6 +17,11 @@ AZURE:
         DISC:
             UNASSIGNED: True
         NIC:
+            UNASSIGNED: True
+        IMAGE:
+            # Image name starts with
+            DELETE_IMAGE:
+
             UNASSIGNED: True
         PUBLIC_IP:
             UNASSIGNED: True
@@ -36,3 +41,4 @@ AZURE:
         GROUP:
             # Resource groups that would be skipped from cleanup
             RG_LIST: []
+        IMAGES: []

settings.yaml.template

@@ -42,6 +42,11 @@ AZURE:
         DISC:
             UNASSIGNED: True
         NIC:
+            UNASSIGNED: True
+        IMAGE:
+            # Image name starts with
+            DELETE_IMAGE:
+
             UNASSIGNED: True
         PUBLIC_IP:
             UNASSIGNED: True
@@ -61,6 +66,7 @@ AZURE:
         GROUP:
             # Resource groups that would be skipped from cleanup
             RG_LIST: []
+        IMAGES: []
 
 AWS:
     AUTH:
@@ -77,6 +83,12 @@ AWS:
         DISC:
             UNASSIGNED: True
         NIC:
+            UNASSIGNED: True
+        # Image name starts with
+        IMAGE:
+            # Image name starts with
+            DELETE_IMAGE:
+
             UNASSIGNED: True
         PUBLIC_IP:
             UNASSIGNED: True
@@ -94,3 +106,4 @@ AWS:
         STACKS:
             # CloudFormations names that would be skipped from cleanup
             STACK_LIST: []
+        IMAGES: []