diff --git a/.github/workflows/cluster_cleanup.yml b/.github/workflows/cluster_cleanup.yml
index abbba1ea1..56f2b11a0 100644
--- a/.github/workflows/cluster_cleanup.yml
+++ b/.github/workflows/cluster_cleanup.yml
@@ -30,6 +30,11 @@ on:
         description: "Resource group to clean up"
         required: true
         default: ops-cli-int-test-rg
+      keyvault_prefix:
+        type: string
+        description: "Prefix of keyvault to delete"
+        default: "opskv"
+        required: false
   # Run every night at midnight (Pacific) to cleanup resources
   schedule:
     - cron: '0 8 * * *'
@@ -71,8 +76,13 @@ jobs:
           subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
       - name: Delete keyvaults
         run: |
-          for vault in $(az keyvault list --query "[?starts_with(name, '${{ env.KEYVAULT_PREFIX }}')].name" -o tsv); do
-            az keyvault delete -n $vault -g ${{ env.RESOURCE_GROUP }} --no-wait
+          for vault in $(az keyvault list -g ${{ env.RESOURCE_GROUP }} --query "[?starts_with(name, '${{ env.KEYVAULT_PREFIX }}')].name" -o tsv); do
+            az keyvault delete -n $vault -g ${{ env.RESOURCE_GROUP }}
+          done
+      - name: Purge keyvaults
+        run: |
+          for vault in $(az keyvault list-deleted --query "[?contains(properties.vaultId, '${{ env.RESOURCE_GROUP }}')] | [?starts_with(name, '${{ env.KEYVAULT_PREFIX }}')].name" -o tsv); do
+            az keyvault purge -n $vault --no-wait
           done
   resource-cleanup:
     needs: [arc-cleanup]
@@ -110,7 +120,7 @@ jobs:
         run: |
           mq_type="Microsoft.IoTOperationsMQ/mq"
           in_cluster_ext_loc="contains(to_string(extendedLocation.name), '${{ env.CLUSTER_PREFIX }}')"
-          
+
           # MQ instance cannot be deleted until all child resources have successfully deleted
           sleep 15s