diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 new file mode 100644 index 00000000000..ef6cea45b40 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 @@ -0,0 +1,33 @@ +--- +# https://docs.cilium.io/en/latest/network/bgp-control-plane/ +apiVersion: cilium.io/v2alpha1 +kind: CiliumBGPPeeringPolicy +metadata: + name: policy +spec: + nodeSelector: + matchLabels: + kubernetes.io/os: linux + virtualRouters: + - localASN: {{ distribution.talos.bgp.local_asn }} + neighbors: + {% if distribution.talos.bgp.peers %} + {% for item in distribution.talos.bgp.peers %} + - peerAddress: "{{ item }}/32" + peerASN: {{ distribution.talos.bgp.peer_asn }} + {% endfor %} + {% else %} + - peerAddress: "{{ nodes.host_network | nthhost(1) }}/32" + peerASN: {{ distribution.talos.bgp.peer_asn }} + {% endif %} + serviceSelector: + matchExpressions: + - {key: somekey, operator: NotIn, values: ['never-used-value']} +--- +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: pool +spec: + cidrs: + - cidr: "${LOADBALANCER_CIDR}" diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 index 083be0f0216..492609d19af 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 @@ -2,7 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - {% if not feature_gates.dual_stack_ipv4_first %} + {% if distribution.talos.bgp.enabled %} + - ./cilium-bgp.yaml + {% endif %} + {% if ( (not distribution.talos.bgp.enabled) and + (feature_gates.dual_stack_ipv4_first) ) %} - ./cilium-l2.yaml {% endif %} - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 index b851eaecacc..035792fbcca 100644 --- a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 @@ -11,3 +11,6 @@ data: {% if feature_gates.dual_stack_ipv4_first %} CLUSTER_CIDR_V6: "{{ cluster.pod_network.split(',')[1] }}" {% endif %} + {% if distribution.talos.bgp.enabled %} + LOADBALANCER_CIDR: "{{ distribution.talos.bgp.loadbalancer_network }}" + {% endif %} diff --git a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 index aeac16ffd58..887d43b65c2 100644 --- a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 @@ -1,6 +1,14 @@ autoDirectNodeRoutes: true +{% if distribution.talos.bgp.enabled %} +bgpControlPlane: + enabled: true +{% endif %} bpf: masquerade: true +cgroup: + automount: + enabled: false + hostRoot: /sys/fs/cgroup cluster: name: home-kubernetes id: 1 @@ -68,7 +76,8 @@ k8sServicePort: 7445 kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 l2announcements: - {% if feature_gates.dual_stack_ipv4_first %} + {% if ( (distribution.talos.bgp.enabled) or + (feature_gates.dual_stack_ipv4_first) ) %} # https://github.com/cilium/cilium/issues/28985 enabled: false {% else %} @@ -101,4 +110,20 @@ dashboards: rollOutCiliumPods: true routingMode: native securityContext: - privileged: true + capabilities: + ciliumAgent: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + cleanCiliumState: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE diff --git a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 index 66114b6ff7e..06f1c01f66c 100644 --- a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 @@ -1,6 +1,14 @@ autoDirectNodeRoutes: true +{% if distribution.talos.bgp.enabled %} +bgpControlPlane: + enabled: true +{% endif %} bpf: masquerade: true +cgroup: + automount: + enabled: false + hostRoot: /sys/fs/cgroup cluster: name: home-kubernetes id: 1 @@ -31,7 +39,8 @@ k8sServicePort: 7445 kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 l2announcements: - {% if feature_gates.dual_stack_ipv4_first %} + {% if ( (distribution.talos.bgp.enabled) or + (feature_gates.dual_stack_ipv4_first) ) %} # https://github.com/cilium/cilium/issues/28985 enabled: false {% else %} @@ -47,4 +56,20 @@ operator: rollOutCiliumPods: true routingMode: native securityContext: - privileged: true + capabilities: + ciliumAgent: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + cleanCiliumState: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE diff --git a/config.sample.yaml b/config.sample.yaml index ca34586e603..bfc238a5605 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -28,6 +28,22 @@ distribution: # # (Optional) Add vlan tag to network master device # # See: https://www.talos.dev/latest/advanced/advanced-networking/#vlans # vlan: 1 + # # (Optional) Use cilium BGP control plane when L2 announcements won't traverse VLAN network segments. + # # Needs a BGP capable router setup with the node IPs as peers. + # # See: https://docs.cilium.io/en/latest/network/bgp-control-plane/ + # bgp: + # enabled: true + # # (Optional) If using multiple BGP peers add them here. + # # Default is .1 derrived from host_network: ['x.x.x.1'] + # peers: [] + # # (Required) Set the BGP Autonomous System Number for the router(s) and nodes. + # # If these match, iBGP will be used. If not, eBGP will be used. + # peer_asn: 64512 # Router(s) AS + # local_asn: 64512 # Node(s) AS + # # (Required) The loadbalancer CIDR for the cluster, this must NOT overlap with any + # # existing networks and is usually a /16 (64K IPs). + # # If you want to use IPv6 check the advanced flags below + # loadbalancer_network: 10.123.0.0/16 # # (Required) Timezone is your IANA formatted timezone (e.g. America/New_York)