From e14858424308f7a5552280d84d2b0390a94c985c Mon Sep 17 00:00:00 2001 From: Charly Molter Date: Thu, 9 Nov 2023 09:56:16 +0100 Subject: [PATCH] feat(kds): enable delta by default (#8262) Signed-off-by: Charly Molter Signed-off-by: Lukasz Dziedziak Co-authored-by: Lukasz Dziedziak --- .circleci/config.yml | 15 +++++++-------- UPGRADE.md | 5 +++++ .../install-control-plane.dump-values.yaml | 4 ++-- deployments/charts/kuma/README.md | 2 +- deployments/charts/kuma/templates/_helpers.tpl | 8 ++++---- deployments/charts/kuma/values.yaml | 4 ++-- docs/generated/kuma-cp.md | 2 +- docs/generated/raw/helm-values.yaml | 4 ++-- docs/generated/raw/kuma-cp.yaml | 2 +- pkg/config/app/kuma-cp/config.go | 2 +- pkg/config/app/kuma-cp/kuma-cp.defaults.yaml | 2 +- .../cp_compatibility_kubernetes_multizone.go | 13 +------------ test/e2e/compatibility/e2e_suite_test.go | 2 +- test/e2e/helm/e2e_suite_test.go | 10 +++++----- test/e2e/helm/kuma_helm_deploy_global_zone.go | 4 ++-- test/e2e/helm/kuma_helm_upgrade.go | 1 - test/framework/config.go | 18 +++++++++--------- 17 files changed, 45 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 43a41545c621..d006f57ae560 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -239,7 +239,7 @@ jobs: description: The CNI networking plugin to use [flannel | calico] type: string default: flannel - deltaKDS: + legacyKDS: description: if should run tests with new implementation of KDS type: boolean default: false @@ -259,7 +259,7 @@ jobs: - {equal: [calico, << parameters.cniNetworkPlugin >>]} - {equal: [kindIpv6, << parameters.k8sVersion >>]} - {equal: [arm64, << parameters.arch >>]} - - {equal: [true, << parameters.deltaKDS >>]} + - {equal: [true, << parameters.legacyKDS >>]} - {equal: [<< pipeline.parameters.first_k8s_version >>, << parameters.k8sVersion >>]} steps: - halt_non_priority_job @@ -336,9 +336,8 @@ jobs: export MAKE_PARAMETERS="-j2" fi - if [[ "<< parameters.deltaKDS >>" == true ]]; then - export KUMA_DELTA_KDS=true - export KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_ENABLED=true + if [[ "<< parameters.legacyKDS >>" == true ]]; then + export KUMA_LEGACY_KDS=true fi if [[ "<< parameters.target >>" == "" ]]; then @@ -494,14 +493,14 @@ workflows: arch: [amd64, arm64] requires: [build, go_cache-<< matrix.arch >>] - e2e: - name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>-delta-kds + name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>-legacy-kds matrix: - alias: delta-kds + alias: legacy-kds parameters: k8sVersion: [<< pipeline.parameters.last_k8s_version >>] target: [multizone] arch: [amd64] - deltaKDS: [true] + legacyKDS: [true] requires: [build, go_cache-amd64] - e2e: name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>-calico diff --git a/UPGRADE.md b/UPGRADE.md index ce7a8ee6260d..7e8c99273905 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -54,6 +54,11 @@ That's why we decided to discontinue TrafficPermission support on the Prometheus We support `v1` resources and `v1.0.0` of `gateway-api`. `v1beta1` resources are still supported but support for these WILL be removed in a future release. +### KDS Delta enabled by default + +KDS Delta is enabled by default. You can fallback to SOTW KDS by setting `KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED=false`. +As a side effect, on kubernetes policies synced will be persisted in the `kuma-system` namespace instead of `default`. + ## Upgrade to `2.4.x` ### Configuration change diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml index 9d00d2dcf205..3a3a61c22624 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml @@ -690,8 +690,8 @@ experimental: tcAttachIface: "" # -- Path where compiled eBPF programs which will be installed can be found programsSourcePath: /kuma/ebpf - # -- If true, it uses new API for resource synchronization - deltaKds: false + # -- If false, it uses legacy API for resource synchronization + deltaKds: true # Postgres' settings for universal control plane on k8s postgres: diff --git a/deployments/charts/kuma/README.md b/deployments/charts/kuma/README.md index 929290754360..b43f46c5dedd 100644 --- a/deployments/charts/kuma/README.md +++ b/deployments/charts/kuma/README.md @@ -204,7 +204,7 @@ A Helm chart for the Kuma Control Plane | experimental.ebpf.cgroupPath | string | `"/sys/fs/cgroup"` | Host's cgroup2 path | | experimental.ebpf.tcAttachIface | string | `""` | Name of the network interface which TC programs should be attached to, we'll try to automatically determine it if empty | | experimental.ebpf.programsSourcePath | string | `"/kuma/ebpf"` | Path where compiled eBPF programs which will be installed can be found | -| experimental.deltaKds | bool | `false` | If true, it uses new API for resource synchronization | +| experimental.deltaKds | bool | `true` | If false, it uses legacy API for resource synchronization | | postgres.port | string | `"5432"` | Postgres port, password should be provided as a secret reference in "controlPlane.secrets" with the Env value "KUMA_STORE_POSTGRES_PASSWORD". Example: controlPlane: secrets: - Secret: postgres-postgresql Key: postgresql-password Env: KUMA_STORE_POSTGRES_PASSWORD | | postgres.tls.mode | string | `"disable"` | Mode of TLS connection. Available values are: "disable", "verifyNone", "verifyCa", "verifyFull" | | postgres.tls.disableSSLSNI | bool | `false` | Whether to disable SNI the postgres `sslsni` option. | diff --git a/deployments/charts/kuma/templates/_helpers.tpl b/deployments/charts/kuma/templates/_helpers.tpl index cb59afea23c7..b209a99bd3dc 100644 --- a/deployments/charts/kuma/templates/_helpers.tpl +++ b/deployments/charts/kuma/templates/_helpers.tpl @@ -275,9 +275,9 @@ env: - name: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_PROGRAMS_SOURCE_PATH value: {{ .Values.experimental.ebpf.programsSourcePath }} {{- end }} -{{- if .Values.experimental.deltaKds }} +{{- if not .Values.experimental.deltaKds }} - name: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED - value: "true" + value: "false" {{- end }} {{- if .Values.controlPlane.tls.kdsZoneClient.skipVerify }} - name: KUMA_MULTIZONE_ZONE_KDS_TLS_SKIP_VERIFY @@ -330,9 +330,9 @@ env: - name: KUMA_MULTIZONE_ZONE_KDS_ROOT_CA_FILE value: /var/run/secrets/kuma.io/kds-client-tls-cert/ca.crt {{- end }} -{{- if .Values.experimental.deltaKds }} +{{- if not .Values.experimental.deltaKds }} - name: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED - value: "true" + value: "false" {{- end }} {{- if .Values.controlPlane.tls.kdsZoneClient.skipVerify }} - name: KUMA_MULTIZONE_ZONE_KDS_TLS_SKIP_VERIFY diff --git a/deployments/charts/kuma/values.yaml b/deployments/charts/kuma/values.yaml index 9d00d2dcf205..3a3a61c22624 100644 --- a/deployments/charts/kuma/values.yaml +++ b/deployments/charts/kuma/values.yaml @@ -690,8 +690,8 @@ experimental: tcAttachIface: "" # -- Path where compiled eBPF programs which will be installed can be found programsSourcePath: /kuma/ebpf - # -- If true, it uses new API for resource synchronization - deltaKds: false + # -- If false, it uses legacy API for resource synchronization + deltaKds: true # Postgres' settings for universal control plane on k8s postgres: diff --git a/docs/generated/kuma-cp.md b/docs/generated/kuma-cp.md index e239b960dae5..16a0264c600b 100644 --- a/docs/generated/kuma-cp.md +++ b/docs/generated/kuma-cp.md @@ -727,7 +727,7 @@ experimental: # format you can downgrade your cp useTagFirstVirtualOutboundModel: false # ENV: KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL # If true, KDS will sync using incremental xDS updates - kdsDeltaEnabled: false # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED + kdsDeltaEnabled: true # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED # List of prefixes that will be used to filter out tags by keys from ingress' available services section. # This can trim the size of the ZoneIngress object significantly. # The drawback is that you cannot use filtered out tags for traffic routing. diff --git a/docs/generated/raw/helm-values.yaml b/docs/generated/raw/helm-values.yaml index 9d00d2dcf205..3a3a61c22624 100644 --- a/docs/generated/raw/helm-values.yaml +++ b/docs/generated/raw/helm-values.yaml @@ -690,8 +690,8 @@ experimental: tcAttachIface: "" # -- Path where compiled eBPF programs which will be installed can be found programsSourcePath: /kuma/ebpf - # -- If true, it uses new API for resource synchronization - deltaKds: false + # -- If false, it uses legacy API for resource synchronization + deltaKds: true # Postgres' settings for universal control plane on k8s postgres: diff --git a/docs/generated/raw/kuma-cp.yaml b/docs/generated/raw/kuma-cp.yaml index d645e81e56f7..78dc053ae78f 100644 --- a/docs/generated/raw/kuma-cp.yaml +++ b/docs/generated/raw/kuma-cp.yaml @@ -724,7 +724,7 @@ experimental: # format you can downgrade your cp useTagFirstVirtualOutboundModel: false # ENV: KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL # If true, KDS will sync using incremental xDS updates - kdsDeltaEnabled: false # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED + kdsDeltaEnabled: true # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED # List of prefixes that will be used to filter out tags by keys from ingress' available services section. # This can trim the size of the ZoneIngress object significantly. # The drawback is that you cannot use filtered out tags for traffic routing. diff --git a/pkg/config/app/kuma-cp/config.go b/pkg/config/app/kuma-cp/config.go index dec4e739a2da..fcff859f0d71 100644 --- a/pkg/config/app/kuma-cp/config.go +++ b/pkg/config/app/kuma-cp/config.go @@ -245,7 +245,7 @@ var DefaultConfig = func() Config { Experimental: ExperimentalConfig{ GatewayAPI: false, KubeOutboundsAsVIPs: true, - KDSDeltaEnabled: false, + KDSDeltaEnabled: true, UseTagFirstVirtualOutboundModel: false, IngressTagFilters: []string{}, KDSEventBasedWatchdog: ExperimentalKDSEventBasedWatchdog{ diff --git a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml index d645e81e56f7..78dc053ae78f 100644 --- a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml +++ b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml @@ -724,7 +724,7 @@ experimental: # format you can downgrade your cp useTagFirstVirtualOutboundModel: false # ENV: KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL # If true, KDS will sync using incremental xDS updates - kdsDeltaEnabled: false # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED + kdsDeltaEnabled: true # ENV: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED # List of prefixes that will be used to filter out tags by keys from ingress' available services section. # This can trim the size of the ZoneIngress object significantly. # The drawback is that you cannot use filtered out tags for traffic routing. diff --git a/test/e2e/compatibility/cp_compatibility_kubernetes_multizone.go b/test/e2e/compatibility/cp_compatibility_kubernetes_multizone.go index 66bf2cc0ca0e..9c06fb8a7ad6 100644 --- a/test/e2e/compatibility/cp_compatibility_kubernetes_multizone.go +++ b/test/e2e/compatibility/cp_compatibility_kubernetes_multizone.go @@ -111,7 +111,7 @@ metadata: // when new resources is created on Zone err = democlient.Install(democlient.WithNamespace(TestNamespace), democlient.WithMesh("default"))(zoneCluster) - // then resource is synchronized to Global + // then resource is synchronized to Global (The namespace here will need to be updated as soon as the minimum version is 2.5.x Expect(err).ToNot(HaveOccurred()) Eventually(func() (string, error) { return k8s.RunKubectlAndGetOutputE(globalCluster.GetTesting(), globalCluster.GetKubectlOptions("default"), "get", "dataplanes") @@ -127,16 +127,5 @@ metadata: WithoutHelmOpt("global.image.tag"), WithHelmChartVersion(Config.SuiteConfig.Compatibility.HelmVersion), }, - ), Entry( - "Sync old global and new zone", - []KumaDeploymentOption{ - WithHelmChartPath(Config.HelmChartName), - WithoutHelmOpt("global.image.tag"), - WithHelmChartVersion(Config.SuiteConfig.Compatibility.HelmVersion), - }, - []KumaDeploymentOption{ - WithInstallationMode(HelmInstallationMode), - WithHelmChartPath(Config.HelmChartPath), - }, )) } diff --git a/test/e2e/compatibility/e2e_suite_test.go b/test/e2e/compatibility/e2e_suite_test.go index 97039e98b6e1..b6dde8711599 100644 --- a/test/e2e/compatibility/e2e_suite_test.go +++ b/test/e2e/compatibility/e2e_suite_test.go @@ -13,4 +13,4 @@ func TestE2E(t *testing.T) { test.RunE2ESpecs(t, "E2E Compatibility Suite") } -var _ = Describe("Test Kubernetes Multizone Compatibility", Label("job-1"), Label("arm-not-supported"), compatibility.CpCompatibilityMultizoneKubernetes) +var _ = Describe("Test Kubernetes Multizone Compatibility", Label("job-1"), compatibility.CpCompatibilityMultizoneKubernetes) diff --git a/test/e2e/helm/e2e_suite_test.go b/test/e2e/helm/e2e_suite_test.go index d5cfe2753e62..db0f1f3c9a2a 100644 --- a/test/e2e/helm/e2e_suite_test.go +++ b/test/e2e/helm/e2e_suite_test.go @@ -14,9 +14,9 @@ func TestE2E(t *testing.T) { } var ( - _ = Describe("Test Zone and Global with Helm chart", Label("job-2"), Label("arm-not-supported"), helm.ZoneAndGlobalWithHelmChart, Ordered) - _ = Describe("Test Zone and Global universal mode with Helm chart", Label("job-0"), Label("arm-not-supported"), helm.ZoneAndGlobalInUniversalModeWithHelmChart, Ordered) - _ = Describe("Test Global and Zone universal mode with Helm chart", Label("job-0"), Label("arm-not-supported"), helm.GlobalAndZoneInUniversalModeWithHelmChart, Ordered) - _ = Describe("Test App deployment with Helm chart", Label("job-2"), Label("arm-not-supported"), helm.AppDeploymentWithHelmChart) - _ = Describe("Test upgrading with Helm chart", Label("job-2"), Label("arm-not-supported"), helm.UpgradingWithHelmChart) + _ = Describe("Test Zone and Global with Helm chart", Label("job-2"), helm.ZoneAndGlobalWithHelmChart, Ordered) + _ = Describe("Test Zone and Global universal mode with Helm chart", Label("job-0"), helm.ZoneAndGlobalInUniversalModeWithHelmChart, Ordered) + _ = Describe("Test Global and Zone universal mode with Helm chart", Label("job-0"), helm.GlobalAndZoneInUniversalModeWithHelmChart, Ordered) + _ = Describe("Test App deployment with Helm chart", Label("job-2"), helm.AppDeploymentWithHelmChart) + _ = Describe("Test upgrading with Helm chart", Label("job-2"), helm.UpgradingWithHelmChart) ) diff --git a/test/e2e/helm/kuma_helm_deploy_global_zone.go b/test/e2e/helm/kuma_helm_deploy_global_zone.go index 7996fabb7239..e35b4e6034c6 100644 --- a/test/e2e/helm/kuma_helm_deploy_global_zone.go +++ b/test/e2e/helm/kuma_helm_deploy_global_zone.go @@ -111,7 +111,7 @@ interCp: // and dataplanes are synced to global Eventually(func() string { - output, err := k8s.RunKubectlAndGetOutputE(c1.GetTesting(), c1.GetKubectlOptions("default"), "get", "dataplanes") + output, err := k8s.RunKubectlAndGetOutputE(c1.GetTesting(), c1.GetKubectlOptions(Config.KumaNamespace), "get", "dataplanes") Expect(err).ToNot(HaveOccurred()) return output }, "5s", "500ms").Should(ContainSubstring("kuma-2-zone.demo-client")) @@ -180,7 +180,7 @@ interCp: podName, err := PodNameOfApp(c2, "demo-client", TestNamespace) Expect(err).ToNot(HaveOccurred()) - dataplaneName := fmt.Sprintf("%s-zone.%s.%s.default", Kuma2, podName, TestNamespace) + dataplaneName := fmt.Sprintf("%s-zone.%s.%s.%s", Kuma2, podName, TestNamespace, Config.KumaNamespace) // when _, err = c1.GetKumactlOptions().RunKumactlAndGetOutput("inspect", "dataplane", dataplaneName, "--type", "config-dump") diff --git a/test/e2e/helm/kuma_helm_upgrade.go b/test/e2e/helm/kuma_helm_upgrade.go index d6e11c47f616..962bbdcc2ea3 100644 --- a/test/e2e/helm/kuma_helm_upgrade.go +++ b/test/e2e/helm/kuma_helm_upgrade.go @@ -69,7 +69,6 @@ func UpgradingWithHelmChart() { // then CRD is upgraded Expect(err).ToNot(HaveOccurred()) Expect(out).To(ContainSubstring("AllowWithShadowDeny")) - // remove this when+then after initialChartVersion is changed to 2.1.x or later }, func() []TableEntry { var out []TableEntry diff --git a/test/framework/config.go b/test/framework/config.go index 40986aebb1bf..80b26f8bb42b 100644 --- a/test/framework/config.go +++ b/test/framework/config.go @@ -59,7 +59,7 @@ type E2eConfig struct { KumaCpConfig KumaCpConfig `json:"kumaCpConfig,omitempty" envconfig:"KUMA_CP_CONFIG"` UniversalE2ELogsPath string `json:"universalE2ELogsPath,omitempty" envconfig:"UNIVERSAL_E2E_LOGS_PATH"` CleanupLogsOnSuccess bool `json:"cleanupLogsOnSuccess,omitempty" envconfig:"CLEANUP_LOGS_ON_SUCCESS"` - KumaDeltaKDS bool `json:"kumaDeltaKDS,omitempty" envconfig:"KUMA_DELTA_KDS"` + KumaLegacyKDS bool `json:"kumaLegacyKDS,omitempty" envconfig:"KUMA_LEGACY_KDS"` SuiteConfig SuiteConfig `json:"suites,omitempty"` } @@ -152,11 +152,11 @@ func (c E2eConfig) AutoConfigure() error { Config.CIDR = "fd00:fd00::/64" } - if Config.KumaDeltaKDS { - Config.KumaCpConfig.Multizone.KubeZone1.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "true" - Config.KumaCpConfig.Multizone.KubeZone2.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "true" - Config.KumaCpConfig.Multizone.UniZone1.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "true" - Config.KumaCpConfig.Multizone.UniZone2.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "true" + if Config.KumaLegacyKDS { + Config.KumaCpConfig.Multizone.KubeZone1.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "false" + Config.KumaCpConfig.Multizone.KubeZone2.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "false" + Config.KumaCpConfig.Multizone.UniZone1.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "false" + Config.KumaCpConfig.Multizone.UniZone2.Envs["KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"] = "false" } Config.Arch = runtime.GOARCH @@ -219,11 +219,11 @@ var defaultConf = E2eConfig{ SuiteConfig: SuiteConfig{ Helm: HelmSuiteConfig{ Versions: []string{ - "2.0.1", + "2.3.3", }, }, Compatibility: CompatibilitySuiteConfig{ - HelmVersion: "1.6.0", + HelmVersion: "2.3.3", }, }, K8sType: KindK8sType, @@ -267,7 +267,7 @@ var defaultConf = E2eConfig{ ZoneIngressApp: "kuma-ingress", UniversalE2ELogsPath: path.Join(os.TempDir(), "e2e"), CleanupLogsOnSuccess: false, - KumaDeltaKDS: false, + KumaLegacyKDS: false, } func init() {