From 3a7d9978cb7cb92b3ec89450c31238c78472042e Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 15:40:39 -0600 Subject: [PATCH 01/14] fix: Use cilium cni install values for Talos --- .../cilium-values-full.partial.yaml.j2 | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 index aeac16ffd58..e520b4f4008 100644 --- a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 @@ -1,6 +1,12 @@ autoDirectNodeRoutes: true bpf: masquerade: true +{% if distribution.type in ["talos"] %} +cgroup: + automount: + enabled: false + hostRoot: /sys/fs/cgroup +{% endif %} cluster: name: home-kubernetes id: 1 @@ -101,4 +107,24 @@ dashboards: rollOutCiliumPods: true routingMode: native securityContext: + {% if distribution.type in ["talos"] %} + capabilities: + ciliumAgent: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + cleanCiliumState: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE + {% else %} privileged: true + {% endif %} From 11f0220187e9a5933aa3591f85dc32c6de2c187d Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 15:42:06 -0600 Subject: [PATCH 02/14] fix: Use cilium cni install values for Talos --- .../cilium-values-init.partial.yaml.j2 | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 index 66114b6ff7e..ca5ccc7ef6a 100644 --- a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 @@ -1,6 +1,12 @@ autoDirectNodeRoutes: true bpf: masquerade: true +{% if distribution.type in ["talos"] %} +cgroup: + automount: + enabled: false + hostRoot: /sys/fs/cgroup +{% endif %} cluster: name: home-kubernetes id: 1 @@ -47,4 +53,24 @@ operator: rollOutCiliumPods: true routingMode: native securityContext: + {% if distribution.type in ["talos"] %} + capabilities: + ciliumAgent: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + cleanCiliumState: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE + {% else %} privileged: true + {% endif %} From a330d51a6f43d1ea411ac1a6ca0bb12c408230e8 Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 16:16:26 -0600 Subject: [PATCH 03/14] feat: Add cilium bgpControlPlane support for Talos --- .../templates/partials/cilium-values-full.partial.yaml.j2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 index e520b4f4008..4ea16d668d3 100644 --- a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 @@ -1,4 +1,8 @@ autoDirectNodeRoutes: true +{% if distribution.talos.bgp.enabled %} +bgpControlPlane: + enabled: true +{% endif %} bpf: masquerade: true {% if distribution.type in ["talos"] %} @@ -74,7 +78,8 @@ k8sServicePort: 7445 kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 l2announcements: - {% if feature_gates.dual_stack_ipv4_first %} + {% if ( (distribution.talos.bgp.enabled) or + (feature_gates.dual_stack_ipv4_first) ) %} # https://github.com/cilium/cilium/issues/28985 enabled: false {% else %} From 245da36b9bc0dc1fddb289ed68e374c5de33f3e4 Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 16:17:40 -0600 Subject: [PATCH 04/14] feat: Add cilium bgpControlPlane support for Talos --- .../templates/partials/cilium-values-init.partial.yaml.j2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 index ca5ccc7ef6a..1635641f963 100644 --- a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 @@ -1,4 +1,8 @@ autoDirectNodeRoutes: true +{% if distribution.talos.bgp.enabled %} +bgpControlPlane: + enabled: true +{% endif %} bpf: masquerade: true {% if distribution.type in ["talos"] %} @@ -37,7 +41,8 @@ k8sServicePort: 7445 kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 l2announcements: - {% if feature_gates.dual_stack_ipv4_first %} + {% if ( (distribution.talos.bgp.enabled) or + (feature_gates.dual_stack_ipv4_first) ) %} # https://github.com/cilium/cilium/issues/28985 enabled: false {% else %} From d297879c8bbc2e3357d80ea3c470593e0c8adfa0 Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 16:20:49 -0600 Subject: [PATCH 05/14] feat: Add LB cidr var for Talos BGPCP --- .../templates/kubernetes/flux/vars/cluster-settings.yaml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 index b851eaecacc..035792fbcca 100644 --- a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 @@ -11,3 +11,6 @@ data: {% if feature_gates.dual_stack_ipv4_first %} CLUSTER_CIDR_V6: "{{ cluster.pod_network.split(',')[1] }}" {% endif %} + {% if distribution.talos.bgp.enabled %} + LOADBALANCER_CIDR: "{{ distribution.talos.bgp.loadbalancer_network }}" + {% endif %} From ab2453a834730fa4da56f87fd06f6fcd532306df Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 16:23:36 -0600 Subject: [PATCH 06/14] feat: Add jinja template for cilium-bgp.yaml --- .../kube-system/cilium/app/cilium-bgp.yaml.j2 | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 new file mode 100644 index 00000000000..0409287b3eb --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 @@ -0,0 +1,34 @@ +--- +# https://docs.cilium.io/en/latest/network/bgp-control-plane/ +apiVersion: cilium.io/v2alpha1 +kind: CiliumBGPPeeringPolicy +metadata: + name: policy +spec: + loadBalancerIPs: true + # NOTE: This might need to be set if you have more than one active NIC on your hosts + # interfaces: + # - ^eno[0-9]+ + nodeSelector: + matchLabels: + kubernetes.io/os: linux + virtualRouters: + - localASN: {{ distribution.talos.bgp.local_asn }} + neighbors: + {% if distribution.talos.bgp.peers %} + {% for item in distribution.talos.bgp.peers %} + - peerAddress: "{{ item }}" + peerASN: {{ distribution.talos.bgp.peer_asn }} + {% endfor %} + {% else %} + - peerAddress: "{{ nodes.host_network | nthhost(1) }}" + peerASN: {{ distribution.talos.bgp.peer_asn }} + {% endif %} +--- +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: pool +spec: + cidrs: + - cidr: "${LOADBALANCER_CIDR}" From 222b56dda1664ecc0f8b0f465362a5a4ca2036bc Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 16:30:40 -0600 Subject: [PATCH 07/14] Add cilium bgpControlPlane support for Talos --- .../apps/kube-system/cilium/app/kustomization.yaml.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 index 083be0f0216..492609d19af 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 @@ -2,7 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - {% if not feature_gates.dual_stack_ipv4_first %} + {% if distribution.talos.bgp.enabled %} + - ./cilium-bgp.yaml + {% endif %} + {% if ( (not distribution.talos.bgp.enabled) and + (feature_gates.dual_stack_ipv4_first) ) %} - ./cilium-l2.yaml {% endif %} - ./helmrelease.yaml From 1fbfa3066dc0b286c4166dd446e51508b1f7fa6d Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 17:10:12 -0600 Subject: [PATCH 08/14] feat: Add cilium bgpControlPlane example config for Talos --- config.sample.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/config.sample.yaml b/config.sample.yaml index ca34586e603..bfc238a5605 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -28,6 +28,22 @@ distribution: # # (Optional) Add vlan tag to network master device # # See: https://www.talos.dev/latest/advanced/advanced-networking/#vlans # vlan: 1 + # # (Optional) Use cilium BGP control plane when L2 announcements won't traverse VLAN network segments. + # # Needs a BGP capable router setup with the node IPs as peers. + # # See: https://docs.cilium.io/en/latest/network/bgp-control-plane/ + # bgp: + # enabled: true + # # (Optional) If using multiple BGP peers add them here. + # # Default is .1 derrived from host_network: ['x.x.x.1'] + # peers: [] + # # (Required) Set the BGP Autonomous System Number for the router(s) and nodes. + # # If these match, iBGP will be used. If not, eBGP will be used. + # peer_asn: 64512 # Router(s) AS + # local_asn: 64512 # Node(s) AS + # # (Required) The loadbalancer CIDR for the cluster, this must NOT overlap with any + # # existing networks and is usually a /16 (64K IPs). + # # If you want to use IPv6 check the advanced flags below + # loadbalancer_network: 10.123.0.0/16 # # (Required) Timezone is your IANA formatted timezone (e.g. America/New_York) From a5f7a2d6e7178ef6450fe3f18581968c36d60ce5 Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 18:50:32 -0600 Subject: [PATCH 09/14] fix: Remove invalid fields from bgp peering policy --- .../apps/kube-system/cilium/app/cilium-bgp.yaml.j2 | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 index 0409287b3eb..cdc4907528d 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 @@ -5,10 +5,6 @@ kind: CiliumBGPPeeringPolicy metadata: name: policy spec: - loadBalancerIPs: true - # NOTE: This might need to be set if you have more than one active NIC on your hosts - # interfaces: - # - ^eno[0-9]+ nodeSelector: matchLabels: kubernetes.io/os: linux @@ -17,11 +13,11 @@ spec: neighbors: {% if distribution.talos.bgp.peers %} {% for item in distribution.talos.bgp.peers %} - - peerAddress: "{{ item }}" + - peerAddress: "{{ item }}/32" peerASN: {{ distribution.talos.bgp.peer_asn }} {% endfor %} {% else %} - - peerAddress: "{{ nodes.host_network | nthhost(1) }}" + - peerAddress: "{{ nodes.host_network | nthhost(1) }}/32" peerASN: {{ distribution.talos.bgp.peer_asn }} {% endif %} --- From 5ad0ae2459ec3b62277440429d2bf64236985a8f Mon Sep 17 00:00:00 2001 From: Wasurerarenai Date: Wed, 7 Feb 2024 19:09:11 -0600 Subject: [PATCH 10/14] fix: Add serviceSelector to bgp peering policy --- .../kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 index cdc4907528d..ef6cea45b40 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 @@ -20,6 +20,9 @@ spec: - peerAddress: "{{ nodes.host_network | nthhost(1) }}/32" peerASN: {{ distribution.talos.bgp.peer_asn }} {% endif %} + serviceSelector: + matchExpressions: + - {key: somekey, operator: NotIn, values: ['never-used-value']} --- apiVersion: cilium.io/v2alpha1 kind: CiliumLoadBalancerIPPool From eaba7670874f23c3db984364004fb13ccaad6f92 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 8 Feb 2024 08:14:22 -0500 Subject: [PATCH 11/14] Update bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 --- .../templates/partials/cilium-values-full.partial.yaml.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 index 4ea16d668d3..76b28dae6a2 100644 --- a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 @@ -112,7 +112,6 @@ dashboards: rollOutCiliumPods: true routingMode: native securityContext: - {% if distribution.type in ["talos"] %} capabilities: ciliumAgent: - CHOWN @@ -130,6 +129,3 @@ securityContext: - NET_ADMIN - SYS_ADMIN - SYS_RESOURCE - {% else %} - privileged: true - {% endif %} From 29bcfe592c9d78c704a1732b1e3f991e2b253344 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 8 Feb 2024 08:14:28 -0500 Subject: [PATCH 12/14] Update bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 --- .../templates/partials/cilium-values-init.partial.yaml.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 index 1635641f963..b9ac80aca19 100644 --- a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 @@ -58,7 +58,6 @@ operator: rollOutCiliumPods: true routingMode: native securityContext: - {% if distribution.type in ["talos"] %} capabilities: ciliumAgent: - CHOWN @@ -76,6 +75,3 @@ securityContext: - NET_ADMIN - SYS_ADMIN - SYS_RESOURCE - {% else %} - privileged: true - {% endif %} From 066869e6d899c97d1916802dbcda13bd79c0b756 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 8 Feb 2024 08:25:43 -0500 Subject: [PATCH 13/14] Update bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 --- bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 index 76b28dae6a2..887d43b65c2 100644 --- a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 @@ -5,12 +5,10 @@ bgpControlPlane: {% endif %} bpf: masquerade: true -{% if distribution.type in ["talos"] %} cgroup: automount: enabled: false hostRoot: /sys/fs/cgroup -{% endif %} cluster: name: home-kubernetes id: 1 From a5761d67c35b9ccc90f1a2844d8fa25e53dcef1d Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 8 Feb 2024 08:26:08 -0500 Subject: [PATCH 14/14] Update bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 --- bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 index b9ac80aca19..06f1c01f66c 100644 --- a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 @@ -5,12 +5,10 @@ bgpControlPlane: {% endif %} bpf: masquerade: true -{% if distribution.type in ["talos"] %} cgroup: automount: enabled: false hostRoot: /sys/fs/cgroup -{% endif %} cluster: name: home-kubernetes id: 1