Add affinity-rules feature to configmap config-deployment

config/core/configmaps/deployment.yaml

@@ -22,7 +22,7 @@ metadata:
     app.kubernetes.io/component: controller
     app.kubernetes.io/version: devel
   annotations:
-    knative.dev/example-checksum: "ed77183a"
+    knative.dev/example-checksum: "1b231253"
 data:
   # This is the Go import path for the binary that is containerized
   # and substituted here.
@@ -91,3 +91,20 @@ data:
     # Sets rootCA for the queue proxy - used by QPOptions
     # If omitted, or empty, no rootCA is added to the golang rootCAs
     queue-sidecar-rootca: ""
+
+    # If set to "true", it automatically configures pod anti-affinity requirements for all Knative services.
+    # It employs the `preferredDuringSchedulingIgnoredDuringExecution` weighted pod affinity term,
+    # aligning with the Knative revision label. It yields the configuration below in all workloads' deployments:
+    # `
+    #   affinity:
+    #     podAntiAffinity:
+    #       preferredDuringSchedulingIgnoredDuringExecution:
+    #       - podAffinityTerm:
+    #           topologyKey: kubernetes.io/hostname
+    #           labelSelector:
+    #             matchLabels:
+    #               serving.knative.dev/revision: {{revision-name}}
+    #         weight: 100
+    # `
+    #
+    # enable-pod-anti-affinity-rule: "true"

pkg/deployment/config.go

@@ -24,7 +24,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/sets"
-
 	cm "knative.dev/pkg/configmap"
 )
 
@@ -68,6 +67,10 @@ const (
 	// qpoptions
 	queueSidecarTokenAudiencesKey = "queue-sidecar-token-audiences"
 	queueSidecarRooCAKey          = "queue-sidecar-rootca"
+
+	enablePodAntiAffinityRule = "enable-pod-anti-affinity-rule"
+
+	EnablePodAntiAffinityRuleDefault = true
 )
 
 var (
@@ -103,6 +106,7 @@ func defaultConfig() *Config {
 		DigestResolutionTimeout:        digestResolutionTimeoutDefault,
 		RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"),
 		QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
+		EnablePodAntiAffinityRule:      EnablePodAntiAffinityRuleDefault,
 	}
 	// The following code is needed for ConfigMap testing.
 	// defaultConfig must match the example in deployment.yaml which includes: `queue-sidecar-token-audiences: ""`
@@ -144,6 +148,8 @@ func NewConfigFromMap(configMap map[string]string) (*Config, error) {
 
 		cm.AsStringSet(queueSidecarTokenAudiencesKey, &nc.QueueSidecarTokenAudiences),
 		cm.AsString(queueSidecarRooCAKey, &nc.QueueSidecarRootCA),
+
+		cm.AsBool(enablePodAntiAffinityRule, &nc.EnablePodAntiAffinityRule),
 	); err != nil {
 		return nil, err
 	}
@@ -214,4 +220,8 @@ type Config struct {
 
 	// QueueSidecarRootCA is a root certificate to be trusted by the queue proxy sidecar  qpoptions.
 	QueueSidecarRootCA string
+
+	// EnablePodAntiAffinityRule is a flag that controls if pod anti-affinity rules will be automatically
+	// applied to the PodSpec of all Knative services.
+	EnablePodAntiAffinityRule bool
 }

pkg/deployment/config_test.go

@@ -81,6 +81,49 @@ func TestControllerConfiguration(t *testing.T) {
 		wantConfig *Config
 		data       map[string]string
 	}{{
+		name: "controller configuration with no pod anti-affinity toggle specified",
+		wantConfig: &Config{
+			RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"),
+			DigestResolutionTimeout:        digestResolutionTimeoutDefault,
+			QueueSidecarImage:              defaultSidecarImage,
+			QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
+			QueueSidecarTokenAudiences:     sets.New(""),
+			ProgressDeadline:               ProgressDeadlineDefault,
+			EnablePodAntiAffinityRule:      EnablePodAntiAffinityRuleDefault,
+		},
+		data: map[string]string{
+			QueueSidecarImageKey: defaultSidecarImage,
+		},
+	}, {
+		name:    "controller configuration with empty string for the pod anti-affinity toggle",
+		wantErr: true,
+		data: map[string]string{
+			QueueSidecarImageKey:      defaultSidecarImage,
+			enablePodAntiAffinityRule: "",
+		},
+	}, {
+		name:    "controller configuration with wrong type for the pod anti-affinity toggle",
+		wantErr: true,
+		data: map[string]string{
+			QueueSidecarImageKey:      defaultSidecarImage,
+			enablePodAntiAffinityRule: "coconut",
+		},
+	}, {
+		name: "controller configuration with the pod anti-affinity toggle on",
+		wantConfig: &Config{
+			RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"),
+			DigestResolutionTimeout:        digestResolutionTimeoutDefault,
+			QueueSidecarImage:              defaultSidecarImage,
+			QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
+			QueueSidecarTokenAudiences:     sets.New(""),
+			ProgressDeadline:               ProgressDeadlineDefault,
+			EnablePodAntiAffinityRule:      true,
+		},
+		data: map[string]string{
+			QueueSidecarImageKey:      defaultSidecarImage,
+			enablePodAntiAffinityRule: "true",
+		},
+	}, {
 		name: "controller configuration with bad registries",
 		wantConfig: &Config{
 			RegistriesSkippingTagResolving: sets.New("ko.local", ""),
@@ -89,6 +132,7 @@ func TestControllerConfiguration(t *testing.T) {
 			QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
 			QueueSidecarTokenAudiences:     sets.New("foo", "bar", "boo-srv"),
 			ProgressDeadline:               ProgressDeadlineDefault,
+			EnablePodAntiAffinityRule:      EnablePodAntiAffinityRuleDefault,
 		},
 		data: map[string]string{
 			QueueSidecarImageKey:              defaultSidecarImage,
@@ -104,6 +148,7 @@ func TestControllerConfiguration(t *testing.T) {
 			QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
 			QueueSidecarTokenAudiences:     sets.New(""),
 			ProgressDeadline:               444 * time.Second,
+			EnablePodAntiAffinityRule:      EnablePodAntiAffinityRuleDefault,
 		},
 		data: map[string]string{
 			QueueSidecarImageKey: defaultSidecarImage,
@@ -118,6 +163,7 @@ func TestControllerConfiguration(t *testing.T) {
 			QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
 			QueueSidecarTokenAudiences:     sets.New(""),
 			ProgressDeadline:               ProgressDeadlineDefault,
+			EnablePodAntiAffinityRule:      EnablePodAntiAffinityRuleDefault,
 		},
 		data: map[string]string{
 			QueueSidecarImageKey:       defaultSidecarImage,
@@ -132,6 +178,7 @@ func TestControllerConfiguration(t *testing.T) {
 			QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
 			QueueSidecarTokenAudiences:     sets.New(""),
 			ProgressDeadline:               ProgressDeadlineDefault,
+			EnablePodAntiAffinityRule:      EnablePodAntiAffinityRuleDefault,
 		},
 		data: map[string]string{
 			QueueSidecarImageKey:              defaultSidecarImage,
@@ -151,6 +198,7 @@ func TestControllerConfiguration(t *testing.T) {
 			QueueSidecarMemoryLimit:             quantity("654m"),
 			QueueSidecarEphemeralStorageLimit:   quantity("321M"),
 			QueueSidecarTokenAudiences:          sets.New(""),
+			EnablePodAntiAffinityRule:           EnablePodAntiAffinityRuleDefault,
 		},
 		data: map[string]string{
 			QueueSidecarImageKey:                   defaultSidecarImage,
@@ -227,6 +275,7 @@ func TestControllerConfiguration(t *testing.T) {
 			QueueSidecarEphemeralStorageRequest: quantity("9M"),
 			QueueSidecarEphemeralStorageLimit:   quantity("10M"),
 			QueueSidecarTokenAudiences:          sets.New(""),
+			EnablePodAntiAffinityRule:           EnablePodAntiAffinityRuleDefault,
 		},
 	}, {
 		name: "newer key case takes priority",
@@ -268,6 +317,7 @@ func TestControllerConfiguration(t *testing.T) {
 			QueueSidecarEphemeralStorageRequest: quantity("20M"),
 			QueueSidecarEphemeralStorageLimit:   quantity("21M"),
 			QueueSidecarTokenAudiences:          sets.New("foo"),
+			EnablePodAntiAffinityRule:           EnablePodAntiAffinityRuleDefault,
 		},
 	}}
 

pkg/reconciler/revision/resources/deploy.go

@@ -150,6 +150,22 @@ func rewriteUserLivenessProbe(p *corev1.Probe, userPort int) {
 	}
 }
 
+func makeDefaultPodAntiAffinity(revisionLabelValue string) *corev1.PodAntiAffinity {
+	return &corev1.PodAntiAffinity{
+		PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{
+			Weight: 100,
+			PodAffinityTerm: corev1.PodAffinityTerm{
+				TopologyKey: corev1.LabelHostname,
+				LabelSelector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						serving.RevisionLabelKey: revisionLabelValue,
+					},
+				},
+			},
+		}},
+	}
+}
+
 func makePodSpec(rev *v1.Revision, cfg *config.Config) (*corev1.PodSpec, error) {
 	queueContainer, err := makeQueueContainer(rev, cfg)
 	tokenVolume := varTokenVolume.DeepCopy()
@@ -210,6 +226,10 @@ func makePodSpec(rev *v1.Revision, cfg *config.Config) (*corev1.PodSpec, error)
 		}
 	}
 
+	if cfg.Deployment.EnablePodAntiAffinityRule && cfg.Features.PodSpecAffinity == apiconfig.Disabled {
+		podSpec.Affinity = &corev1.Affinity{PodAntiAffinity: makeDefaultPodAntiAffinity(rev.Name)}
+	}
+
 	return podSpec, nil
 }
 

pkg/reconciler/revision/resources/deploy_test.go

@@ -205,6 +205,20 @@ var (
 		EnableServiceLinks:            ptr.Bool(false),
 	}
 
+	defaultPodAntiAffinityRules = &corev1.PodAntiAffinity{
+		PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{
+			Weight: 100,
+			PodAffinityTerm: corev1.PodAffinityTerm{
+				TopologyKey: "kubernetes.io/hostname",
+				LabelSelector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"serving.knative.dev/revision": "bar",
+					},
+				},
+			},
+		}},
+	}
+
 	maxUnavailable    = intstr.FromInt(0)
 	defaultDeployment = &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
@@ -1409,6 +1423,89 @@ func TestMakePodSpec(t *testing.T) {
 					withEnvVar("SERVING_READINESS_PROBE", `[{"httpGet":{"path":"/","port":8080,"host":"127.0.0.1","scheme":"HTTP"}},{"httpGet":{"path":"/","port":8090,"host":"127.0.0.1","scheme":"HTTP"}}]`),
 				),
 			}),
+	}, {
+		name: "with default pod anti-affinity rules",
+		rev: revision("bar", "foo",
+			withContainers([]corev1.Container{{
+				Name:           servingContainerName,
+				Image:          "busybox",
+				ReadinessProbe: withTCPReadinessProbe(v1.DefaultUserPort),
+			}}),
+			WithContainerStatuses([]v1.ContainerStatus{{
+				ImageDigest: "busybox@sha256:deadbeef",
+			}}),
+		),
+		fc: apicfg.Features{
+			PodSpecAffinity: apicfg.Disabled,
+		},
+		dc: deployment.Config{
+			EnablePodAntiAffinityRule: true,
+		},
+		want: podSpec(
+			[]corev1.Container{
+				servingContainer(func(container *corev1.Container) {
+					container.Image = "busybox@sha256:deadbeef"
+				}),
+				queueContainer(),
+			},
+			func(p *corev1.PodSpec) {
+				p.Affinity = &corev1.Affinity{
+					PodAntiAffinity: defaultPodAntiAffinityRules,
+				}
+			},
+		),
+	}, {
+		name: "with pod anti-affinity rules toggle off",
+		rev: revision("bar", "foo",
+			withContainers([]corev1.Container{{
+				Name:           servingContainerName,
+				Image:          "busybox",
+				ReadinessProbe: withTCPReadinessProbe(v1.DefaultUserPort),
+			}}),
+			WithContainerStatuses([]v1.ContainerStatus{{
+				ImageDigest: "busybox@sha256:deadbeef",
+			}}),
+		),
+		fc: apicfg.Features{
+			PodSpecAffinity: apicfg.Disabled,
+		},
+		dc: deployment.Config{
+			EnablePodAntiAffinityRule: false,
+		},
+		want: podSpec(
+			[]corev1.Container{
+				servingContainer(func(container *corev1.Container) {
+					container.Image = "busybox@sha256:deadbeef"
+				}),
+				queueContainer(),
+			},
+		),
+	}, {
+		name: "with pod anti-affinity rules toggle on for both users and operators",
+		rev: revision("bar", "foo",
+			withContainers([]corev1.Container{{
+				Name:           servingContainerName,
+				Image:          "busybox",
+				ReadinessProbe: withTCPReadinessProbe(v1.DefaultUserPort),
+			}}),
+			WithContainerStatuses([]v1.ContainerStatus{{
+				ImageDigest: "busybox@sha256:deadbeef",
+			}}),
+		),
+		fc: apicfg.Features{
+			PodSpecAffinity: apicfg.Enabled,
+		},
+		dc: deployment.Config{
+			EnablePodAntiAffinityRule: true,
+		},
+		want: podSpec(
+			[]corev1.Container{
+				servingContainer(func(container *corev1.Container) {
+					container.Image = "busybox@sha256:deadbeef"
+				}),
+				queueContainer(),
+			},
+		),
 	}}
 
 	for _, test := range tests {