apps-sc: add policy to reject local storage emptydir

config/common-config.yaml

@@ -923,6 +923,11 @@ opa:
     enabled: true
     enforcement: deny
 
+  ## Enable rule that warns about emptydir with local storage, which can prevent cluster autoscaler from evicting when scaling down
+  rejectLocalStorageEmptyDir:
+    enabled: false
+    enforcement: warn
+
 trivy:
   enabled: true
   # comma separated list of namespaces (or glob patterns) to be excluded from scanning

config/k8s-installers/capi/wc-config.yaml

@@ -0,0 +1,4 @@
+opa:
+  rejectLocalStorageEmptyDir:
+    # In cluster api cluster autoscaler is regularly enabled, which is when we want to have this enabled
+    enabled: true

config/schemas/config.yaml

@@ -3624,6 +3624,32 @@ properties:
               deny: Deny actions violating the constraint.
               warn: Warn actions violating the constraint.
               dryrun: Dryrun actions violating the constraint.
+      rejectLocalStorageEmptyDir:
+        title: Safeguard Reject Local Storage EmptyDir
+        description: |-
+          Configure constraint to reject usage of local storage emptydir.
+
+          > [!note]
+          > See [the dev docs](https://elastisys.io/welkin/user-guide/safeguards/enforce-no-local-storage-emptydir/) for context.
+        type: object
+        additionalProperties: false
+        properties:
+          enabled:
+            title: Safeguard Rejecting Local Storage EmptyDir Enabled
+            type: boolean
+            default: true
+          enforcement:
+            title: Safeguard Reject Local Storage EmptyDir Enforcement
+            type: string
+            default: warn
+            enum:
+              - deny
+              - warn
+              - dryrun
+            meta:enum:
+              deny: Deny actions violating the constraint.
+              warn: Warn actions violating the constraint.
+              dryrun: Dryrun actions violating the constraint.
       networkPolicies:
         title: Safeguard Network Policies
         description: |-

helmfile.d/charts/gatekeeper/constraints/templates/reject-local-storage-empty-dir/constraint.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.rejectLocalStorageEmptyDir.enable -}}
+apiVersion: constraints.gatekeeper.sh/v1alpha1
+kind: K8sRejectLocalStorageEmptyDir
+metadata:
+  name: elastisys-reject-local-storage-emptydir
+spec:
+  enforcementAction: {{ .Values.rejectLocalStorageEmptyDir.enforcementAction }}
+  match:
+    kinds:
+      - apiGroups: [""]
+        kinds: ["Pod", "ReplicationController"]
+      - apiGroups: ["apps"]
+        kinds: ["Deployment", "StatefulSet", "DaemonSet", "ReplicaSet"]
+      - apiGroups: ["extensions"]
+        kinds: ["Deployment", "StatefulSet", "DaemonSet", "ReplicaSet"]
+      - apiGroups: ["batch"]
+        kinds: ["Job", "CronJob"]
+    excludedNamespaces: ["kube-system", "kube-public", "kube-node-lease", "calico-system"]
+    namespaceSelector:
+      matchExpressions:
+      - key: owner
+        operator: NotIn
+        values:
+        - operator
+  parameters:
+    volumeAnnotation: cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes
+    podAnnotation: cluster-autoscaler.kubernetes.io/safe-to-evict
+{{- end }}

helmfile.d/charts/gatekeeper/constraints/values.yaml

@@ -25,6 +25,9 @@ preventAccidentalDeletion:
 disallowLocalhostSeccomp:
     enable: false
     enforcementAction: deny
+rejectLocalStorageEmptyDir:
+    enable: false
+    enforcementAction: warn
 
 imageRegistryURL: registry.example.com
 

helmfile.d/charts/gatekeeper/templates/policies/reject-local-storage-empty-dir.rego

@@ -0,0 +1,90 @@
+package k8srejectlocalstorageemptydir
+import future.keywords.in
+
+# violation if volume has no medium.
+violation[{"msg": msg}] {
+    volume := get_volumes[_]
+    missing(volume.emptyDir, "medium")
+    not check_volume_in_annotation(get_metadata, volume)
+    not check_pod_annotation(get_metadata)
+    msg := sprintf("The volume <%v> emptyDir is using local storage emptyDir. This can prevent autoscaler from scaling down a node where this is running. Read more about this and possible solutions at https://elastisys.io/welkin/user-guide/safeguards/enforce-no-local-storage-emptydir/",[volume])
+}
+
+# violation if medium is not Memory.
+violation[{"msg": msg}] {
+    volume := get_volumes[_]
+    volume.emptyDir.medium != "Memory"
+    not check_volume_in_annotation(get_metadata, volume)
+    not check_pod_annotation(get_metadata)
+    msg := sprintf("The volume <%v> emptyDir is using local storage emptyDir. This can prevent autoscaler from scaling down a node where this is running. Read more about this and possible solutions at https://elastisys.io/welkin/user-guide/safeguards/enforce-no-local-storage-emptydir/",[volume])
+}
+
+# Get volumes for "Pods"
+get_volumes = res {
+    input.review.object.kind == "Pod"
+    res := input.review.object.spec.volumes
+}
+
+# Get volumes for resources that use pod templates.
+get_volumes = res {
+    kinds := [
+        "Deployment",
+        "StatefulSet",
+        "DaemonSet",
+        "ReplicaSet",
+        "Job",
+        "ReplicationController"
+    ]
+    input.review.object.kind == kinds[_]
+
+    res := input.review.object.spec.template.spec.volumes
+}
+
+# Get volumes for "CronJobs"
+get_volumes = res {
+    input.review.object.kind == "CronJob"
+    res := input.review.object.spec.jobTemplate.spec.template.spec.volumes
+}
+
+# Get metadata for "Pods"
+get_metadata = res {
+    input.review.object.kind == "Pod"
+    res := input.review.object.metadata
+}
+
+# Get metadata for resources that use pod templates.
+get_metadata = res {
+    kinds := [
+        "Deployment",
+        "StatefulSet",
+        "DaemonSet",
+        "ReplicaSet",
+        "Job",
+        "ReplicationController"
+    ]
+    input.review.object.kind == kinds[_]
+
+    res := input.review.object.spec.template.metadata
+}
+
+# Get metadata for "CronJobs"
+get_metadata = res {
+    input.review.object.kind == "CronJob"
+    res := input.review.object.spec.jobTemplate.spec.template.metadata
+}
+
+# Field missing if it does not exist in the object
+missing(obj, field) {
+    not obj[field]
+}
+
+check_volume_in_annotation(metadata, volume) {
+    some annotation_key, annotation_value in metadata.annotations
+    annotation_key == input.parameters.volumeAnnotation
+
+    split(annotation_value, ",")[_] == volume.name
+}
+
+check_pod_annotation(metadata) {
+    metadata.annotations[input.parameters.podAnnotation] == "true"
+}