feat(kds): enable delta by default (#8262)

Signed-off-by: Charly Molter <[email protected]>
Signed-off-by: Lukasz Dziedziak <[email protected]>
Co-authored-by: Lukasz Dziedziak <[email protected]>

.circleci/config.yml

@@ -239,7 +239,7 @@ jobs:
         description: The CNI networking plugin to use [flannel | calico]
         type: string
         default: flannel
-      deltaKDS:
+      legacyKDS:
         description: if should run tests with new implementation of KDS
         type: boolean
         default: false
@@ -259,7 +259,7 @@ jobs:
               - {equal: [calico, << parameters.cniNetworkPlugin >>]}
               - {equal: [kindIpv6, << parameters.k8sVersion >>]}
               - {equal: [arm64, << parameters.arch >>]}
-              - {equal: [true, << parameters.deltaKDS >>]}
+              - {equal: [true, << parameters.legacyKDS >>]}
               - {equal: [<< pipeline.parameters.first_k8s_version >>, << parameters.k8sVersion >>]}
           steps:
             - halt_non_priority_job
@@ -336,9 +336,8 @@ jobs:
               export MAKE_PARAMETERS="-j2"
             fi
 
-            if [[ "<< parameters.deltaKDS >>" == true ]]; then
-              export KUMA_DELTA_KDS=true
-              export KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_ENABLED=true
+            if [[ "<< parameters.legacyKDS >>" == true ]]; then
+              export KUMA_LEGACY_KDS=true
             fi
 
             if [[ "<< parameters.target >>" == "" ]]; then
@@ -494,14 +493,14 @@ workflows:
               arch: [amd64, arm64]
           requires: [build, go_cache-<< matrix.arch >>]
       - e2e:
-          name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>-delta-kds
+          name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>-legacy-kds
           matrix:
-            alias: delta-kds
+            alias: legacy-kds
             parameters:
               k8sVersion: [<< pipeline.parameters.last_k8s_version >>]
               target: [multizone]
               arch: [amd64]
-              deltaKDS: [true]
+              legacyKDS: [true]
           requires: [build, go_cache-amd64]
       - e2e:
           name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>-calico

UPGRADE.md

@@ -54,6 +54,11 @@ That's why we decided to discontinue TrafficPermission support on the Prometheus
 We support `v1` resources and `v1.0.0` of `gateway-api`. `v1beta1` resources are
 still supported but support for these WILL be removed in a future release.
 
+### KDS Delta enabled by default
+
+KDS Delta is enabled by default. You can fallback to SOTW KDS by setting `KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED=false`.
+As a side effect, on kubernetes policies synced will be persisted in the `kuma-system` namespace instead of `default`.
+
 ## Upgrade to `2.4.x`
 
 ### Configuration change

app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml

@@ -690,8 +690,8 @@ experimental:
     tcAttachIface: ""
     # -- Path where compiled eBPF programs which will be installed can be found
     programsSourcePath: /kuma/ebpf
-  # -- If true, it uses new API for resource synchronization
-  deltaKds: false
+  # -- If false, it uses legacy API for resource synchronization
+  deltaKds: true
 
 # Postgres' settings for universal control plane on k8s
 postgres:

deployments/charts/kuma/README.md

@@ -204,7 +204,7 @@ A Helm chart for the Kuma Control Plane
 | experimental.ebpf.cgroupPath | string | `"/sys/fs/cgroup"` | Host's cgroup2 path |
 | experimental.ebpf.tcAttachIface | string | `""` | Name of the network interface which TC programs should be attached to, we'll try to automatically determine it if empty |
 | experimental.ebpf.programsSourcePath | string | `"/kuma/ebpf"` | Path where compiled eBPF programs which will be installed can be found |
-| experimental.deltaKds | bool | `false` | If true, it uses new API for resource synchronization |
+| experimental.deltaKds | bool | `true` | If false, it uses legacy API for resource synchronization |
 | postgres.port | string | `"5432"` | Postgres port, password should be provided as a secret reference in "controlPlane.secrets" with the Env value "KUMA_STORE_POSTGRES_PASSWORD". Example: controlPlane:   secrets:     - Secret: postgres-postgresql       Key: postgresql-password       Env: KUMA_STORE_POSTGRES_PASSWORD |
 | postgres.tls.mode | string | `"disable"` | Mode of TLS connection. Available values are: "disable", "verifyNone", "verifyCa", "verifyFull" |
 | postgres.tls.disableSSLSNI | bool | `false` | Whether to disable SNI the postgres `sslsni` option. |

deployments/charts/kuma/templates/_helpers.tpl

@@ -275,9 +275,9 @@ env:
 - name: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_PROGRAMS_SOURCE_PATH
   value: {{ .Values.experimental.ebpf.programsSourcePath }}
 {{- end }}
-{{- if .Values.experimental.deltaKds }}
+{{- if not .Values.experimental.deltaKds }}
 - name: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
-  value: "true"
+  value: "false"
 {{- end }}
 {{- if .Values.controlPlane.tls.kdsZoneClient.skipVerify }}
 - name: KUMA_MULTIZONE_ZONE_KDS_TLS_SKIP_VERIFY
@@ -330,9 +330,9 @@ env:
 - name: KUMA_MULTIZONE_ZONE_KDS_ROOT_CA_FILE
   value: /var/run/secrets/kuma.io/kds-client-tls-cert/ca.crt
 {{- end }}
-{{- if .Values.experimental.deltaKds }}
+{{- if not .Values.experimental.deltaKds }}
 - name: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
-  value: "true"
+  value: "false"
 {{- end }}
 {{- if .Values.controlPlane.tls.kdsZoneClient.skipVerify }}
 - name: KUMA_MULTIZONE_ZONE_KDS_TLS_SKIP_VERIFY

deployments/charts/kuma/values.yaml

@@ -690,8 +690,8 @@ experimental:
     tcAttachIface: ""
     # -- Path where compiled eBPF programs which will be installed can be found
     programsSourcePath: /kuma/ebpf
-  # -- If true, it uses new API for resource synchronization
-  deltaKds: false
+  # -- If false, it uses legacy API for resource synchronization
+  deltaKds: true
 
 # Postgres' settings for universal control plane on k8s
 postgres:

docs/generated/kuma-cp.md

@@ -727,7 +727,7 @@ experimental:
   # format you can downgrade your cp
   useTagFirstVirtualOutboundModel: false # ENV: KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL
   # If true, KDS will sync using incremental xDS updates
-  kdsDeltaEnabled: false # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
+  kdsDeltaEnabled: true # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
   # List of prefixes that will be used to filter out tags by keys from ingress' available services section.
   # This can trim the size of the ZoneIngress object significantly.
   # The drawback is that you cannot use filtered out tags for traffic routing.

docs/generated/raw/helm-values.yaml

@@ -690,8 +690,8 @@ experimental:
     tcAttachIface: ""
     # -- Path where compiled eBPF programs which will be installed can be found
     programsSourcePath: /kuma/ebpf
-  # -- If true, it uses new API for resource synchronization
-  deltaKds: false
+  # -- If false, it uses legacy API for resource synchronization
+  deltaKds: true
 
 # Postgres' settings for universal control plane on k8s
 postgres:

docs/generated/raw/kuma-cp.yaml

@@ -724,7 +724,7 @@ experimental:
   # format you can downgrade your cp
   useTagFirstVirtualOutboundModel: false # ENV: KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL
   # If true, KDS will sync using incremental xDS updates
-  kdsDeltaEnabled: false # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
+  kdsDeltaEnabled: true # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
   # List of prefixes that will be used to filter out tags by keys from ingress' available services section.
   # This can trim the size of the ZoneIngress object significantly.
   # The drawback is that you cannot use filtered out tags for traffic routing.

pkg/config/app/kuma-cp/config.go

@@ -245,7 +245,7 @@ var DefaultConfig = func() Config {
 		Experimental: ExperimentalConfig{
 			GatewayAPI:                      false,
 			KubeOutboundsAsVIPs:             true,
-			KDSDeltaEnabled:                 false,
+			KDSDeltaEnabled:                 true,
 			UseTagFirstVirtualOutboundModel: false,
 			IngressTagFilters:               []string{},
 			KDSEventBasedWatchdog: ExperimentalKDSEventBasedWatchdog{

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -724,7 +724,7 @@ experimental:
   # format you can downgrade your cp
   useTagFirstVirtualOutboundModel: false # ENV: KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL
   # If true, KDS will sync using incremental xDS updates
-  kdsDeltaEnabled: false # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
+  kdsDeltaEnabled: true # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
   # List of prefixes that will be used to filter out tags by keys from ingress' available services section.
   # This can trim the size of the ZoneIngress object significantly.
   # The drawback is that you cannot use filtered out tags for traffic routing.

test/e2e/compatibility/cp_compatibility_kubernetes_multizone.go

@@ -111,7 +111,7 @@ metadata:
 		// when new resources is created on Zone
 		err = democlient.Install(democlient.WithNamespace(TestNamespace), democlient.WithMesh("default"))(zoneCluster)
 
-		// then resource is synchronized to Global
+		// then resource is synchronized to Global (The namespace here will need to be updated as soon as the minimum version is 2.5.x
 		Expect(err).ToNot(HaveOccurred())
 		Eventually(func() (string, error) {
 			return k8s.RunKubectlAndGetOutputE(globalCluster.GetTesting(), globalCluster.GetKubectlOptions("default"), "get", "dataplanes")
@@ -127,16 +127,5 @@ metadata:
 			WithoutHelmOpt("global.image.tag"),
 			WithHelmChartVersion(Config.SuiteConfig.Compatibility.HelmVersion),
 		},
-	), Entry(
-		"Sync old global and new zone",
-		[]KumaDeploymentOption{
-			WithHelmChartPath(Config.HelmChartName),
-			WithoutHelmOpt("global.image.tag"),
-			WithHelmChartVersion(Config.SuiteConfig.Compatibility.HelmVersion),
-		},
-		[]KumaDeploymentOption{
-			WithInstallationMode(HelmInstallationMode),
-			WithHelmChartPath(Config.HelmChartPath),
-		},
 	))
 }

test/e2e/compatibility/e2e_suite_test.go

@@ -13,4 +13,4 @@ func TestE2E(t *testing.T) {
 	test.RunE2ESpecs(t, "E2E Compatibility Suite")
 }
 
-var _ = Describe("Test Kubernetes Multizone Compatibility", Label("job-1"), Label("arm-not-supported"), compatibility.CpCompatibilityMultizoneKubernetes)
+var _ = Describe("Test Kubernetes Multizone Compatibility", Label("job-1"), compatibility.CpCompatibilityMultizoneKubernetes)

test/e2e/helm/e2e_suite_test.go

@@ -14,9 +14,9 @@ func TestE2E(t *testing.T) {
 }
 
 var (
-	_ = Describe("Test Zone and Global with Helm chart", Label("job-2"), Label("arm-not-supported"), helm.ZoneAndGlobalWithHelmChart, Ordered)
-	_ = Describe("Test Zone and Global universal mode with Helm chart", Label("job-0"), Label("arm-not-supported"), helm.ZoneAndGlobalInUniversalModeWithHelmChart, Ordered)
-	_ = Describe("Test Global and Zone universal mode with Helm chart", Label("job-0"), Label("arm-not-supported"), helm.GlobalAndZoneInUniversalModeWithHelmChart, Ordered)
-	_ = Describe("Test App deployment with Helm chart", Label("job-2"), Label("arm-not-supported"), helm.AppDeploymentWithHelmChart)
-	_ = Describe("Test upgrading with Helm chart", Label("job-2"), Label("arm-not-supported"), helm.UpgradingWithHelmChart)
+	_ = Describe("Test Zone and Global with Helm chart", Label("job-2"), helm.ZoneAndGlobalWithHelmChart, Ordered)
+	_ = Describe("Test Zone and Global universal mode with Helm chart", Label("job-0"), helm.ZoneAndGlobalInUniversalModeWithHelmChart, Ordered)
+	_ = Describe("Test Global and Zone universal mode with Helm chart", Label("job-0"), helm.GlobalAndZoneInUniversalModeWithHelmChart, Ordered)
+	_ = Describe("Test App deployment with Helm chart", Label("job-2"), helm.AppDeploymentWithHelmChart)
+	_ = Describe("Test upgrading with Helm chart", Label("job-2"), helm.UpgradingWithHelmChart)
 )

test/e2e/helm/kuma_helm_deploy_global_zone.go

@@ -111,7 +111,7 @@ interCp:
 
 		// and dataplanes are synced to global
 		Eventually(func() string {
-			output, err := k8s.RunKubectlAndGetOutputE(c1.GetTesting(), c1.GetKubectlOptions("default"), "get", "dataplanes")
+			output, err := k8s.RunKubectlAndGetOutputE(c1.GetTesting(), c1.GetKubectlOptions(Config.KumaNamespace), "get", "dataplanes")
 			Expect(err).ToNot(HaveOccurred())
 			return output
 		}, "5s", "500ms").Should(ContainSubstring("kuma-2-zone.demo-client"))
@@ -180,7 +180,7 @@ interCp:
 
 		podName, err := PodNameOfApp(c2, "demo-client", TestNamespace)
 		Expect(err).ToNot(HaveOccurred())
-		dataplaneName := fmt.Sprintf("%s-zone.%s.%s.default", Kuma2, podName, TestNamespace)
+		dataplaneName := fmt.Sprintf("%s-zone.%s.%s.%s", Kuma2, podName, TestNamespace, Config.KumaNamespace)
 
 		// when
 		_, err = c1.GetKumactlOptions().RunKumactlAndGetOutput("inspect", "dataplane", dataplaneName, "--type", "config-dump")

test/e2e/helm/kuma_helm_upgrade.go

@@ -69,7 +69,6 @@ func UpgradingWithHelmChart() {
 			// then CRD is upgraded
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).To(ContainSubstring("AllowWithShadowDeny"))
-			// remove this when+then after initialChartVersion is changed to 2.1.x or later
 		},
 		func() []TableEntry {
 			var out []TableEntry

test/framework/config.go

@@ -59,7 +59,7 @@ type E2eConfig struct {
 	KumaCpConfig                  KumaCpConfig      `json:"kumaCpConfig,omitempty" envconfig:"KUMA_CP_CONFIG"`
 	UniversalE2ELogsPath          string            `json:"universalE2ELogsPath,omitempty" envconfig:"UNIVERSAL_E2E_LOGS_PATH"`
 	CleanupLogsOnSuccess          bool              `json:"cleanupLogsOnSuccess,omitempty" envconfig:"CLEANUP_LOGS_ON_SUCCESS"`
-	KumaDeltaKDS                  bool              `json:"kumaDeltaKDS,omitempty" envconfig:"KUMA_DELTA_KDS"`
+	KumaLegacyKDS                 bool              `json:"kumaLegacyKDS,omitempty" envconfig:"KUMA_LEGACY_KDS"`
 
 	SuiteConfig SuiteConfig `json:"suites,omitempty"`
 }
@@ -152,11 +152,11 @@ func (c E2eConfig) AutoConfigure() error {
 		Config.CIDR = "fd00:fd00::/64"
 	}
 
-	if Config.KumaDeltaKDS {
-		Config.KumaCpConfig.Multizone.KubeZone1.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "true"
-		Config.KumaCpConfig.Multizone.KubeZone2.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "true"
-		Config.KumaCpConfig.Multizone.UniZone1.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "true"
-		Config.KumaCpConfig.Multizone.UniZone2.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "true"
+	if Config.KumaLegacyKDS {
+		Config.KumaCpConfig.Multizone.KubeZone1.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "false"
+		Config.KumaCpConfig.Multizone.KubeZone2.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "false"
+		Config.KumaCpConfig.Multizone.UniZone1.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "false"
+		Config.KumaCpConfig.Multizone.UniZone2.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "false"
 	}
 
 	Config.Arch = runtime.GOARCH
@@ -219,11 +219,11 @@ var defaultConf = E2eConfig{
 	SuiteConfig: SuiteConfig{
 		Helm: HelmSuiteConfig{
 			Versions: []string{
-				"2.0.1",
+				"2.3.3",
 			},
 		},
 		Compatibility: CompatibilitySuiteConfig{
-			HelmVersion: "1.6.0",
+			HelmVersion: "2.3.3",
 		},
 	},
 	K8sType:                      KindK8sType,
@@ -267,7 +267,7 @@ var defaultConf = E2eConfig{
 	ZoneIngressApp:       "kuma-ingress",
 	UniversalE2ELogsPath: path.Join(os.TempDir(), "e2e"),
 	CleanupLogsOnSuccess: false,
-	KumaDeltaKDS:         false,
+	KumaLegacyKDS:        false,
 }
 
 func init() {