diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 759f393c28..c27ac74e85 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -45,14 +45,13 @@ steps: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl kubectl cluster-info kubectl get po -owide -A - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 - chmod 700 get_helm.sh - ./get_helm.sh echo "deploy Cilium ConfigMap" kubectl apply -f cilium/configmap.yaml + kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml echo "install Cilium" - helm repo add cilium https://helm.cilium.io/ - helm install cilium cilium/cilium --version 1.12.5 --namespace kube-system -f cilium/cilium_helm_values.yaml + kubectl apply -f test/integration/manifests/cilium/cilium-agent + kubectl apply -f test/integration/manifests/cilium/cilium-operator + kubectl get po -owide -A name: "installCilium" displayName: "Install Cilium" @@ -108,6 +107,14 @@ steps: pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output" condition: always() + - script: | + echo "validate pod IP assignment and check systemd-networkd restart" + kubectl apply -f hack/manifests/hostprocess.yaml + kubectl get pod -owide -A + bash hack/scripts/validate_state.sh + name: "validatePods" + displayName: "Validate Pods" + - script: | echo "Run Service Conformance E2E" export PATH=${PATH}:/usr/local/bin/gsutil diff --git a/.pipelines/singletenancy/overlay/overlay-e2e-step-template.yaml b/.pipelines/singletenancy/overlay/overlay-e2e-step-template.yaml index aa324703f4..027e07718d 100644 --- a/.pipelines/singletenancy/overlay/overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/overlay/overlay-e2e-step-template.yaml @@ -45,14 +45,13 @@ steps: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl kubectl cluster-info kubectl get po -owide -A - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 - chmod 700 get_helm.sh - ./get_helm.sh echo "deploy Cilium ConfigMap" kubectl apply -f cilium/configmap.yaml + kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml echo "install Cilium onto Overlay Cluster" - helm repo add cilium https://helm.cilium.io/ - helm install cilium cilium/cilium --version 1.12.5 --namespace kube-system -f cilium/cilium_helm_values.yaml + kubectl apply -f test/integration/manifests/cilium/cilium-agent + kubectl apply -f test/integration/manifests/cilium/cilium-operator + kubectl get po -owide -A name: "installCilium" displayName: "Install Cilium on AKS Overlay" @@ -114,6 +113,14 @@ steps: pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output" condition: always() + - script: | + echo "validate pod IP assignment and check systemd-networkd restart" + kubectl apply -f hack/manifests/hostprocess.yaml + kubectl get pod -owide -A + bash hack/scripts/validate_state.sh + name: "validatePods" + displayName: "Validate Pods" + - script: | echo "Run Service Conformance E2E" export PATH=${PATH}:/usr/local/bin/gsutil diff --git a/hack/scripts/validate_state.sh b/hack/scripts/validate_state.sh index 0ec48f1bd3..c9b8e49595 100644 --- a/hack/scripts/validate_state.sh +++ b/hack/scripts/validate_state.sh @@ -17,6 +17,10 @@ do echo "Node internal ip: $node_ip" privileged_pod=$(kubectl get pods -n kube-system -l app=privileged-daemonset -o wide | grep "$node_name" | awk '{print $1}') echo "privileged pod : $privileged_pod" + if [ "$privileged_pod" == '' ]; then + kubectl describe daemonset privileged-daemonset -n kube-system + exit 1 + fi while ! [ -s "azure_endpoints.json" ] do echo "trying to get the azure_endpoints" @@ -34,6 +38,16 @@ do sleep 10 done + cns_pod=$(kubectl get pod -l k8s-app=azure-cns -n kube-system -o wide | grep "$node_name" | awk '{print $1}') + echo "azure-cns pod : $cns_pod" + + while ! [ -s "cns_endpoints.json" ] + do + echo "trying to get the cns_endpoints" + kubectl exec -it "$cns_pod" -n kube-system -- curl localhost:10090/debug/ipaddresses -d '{"IPConfigStateFilter":["Assigned"]}' > cns_endpoints.json + sleep 10 + done + total_pods=$(kubectl get pods --all-namespaces -o wide --field-selector spec.nodeName="$node_name",status.phase=Running --output json) echo "Checking if there are any pods with no ips" @@ -60,7 +74,7 @@ do echo "Number of azure endpoint ips : $num_of_azure_endpoint_ips" if [ "$num_of_pod_ips" != "$num_of_azure_endpoint_ips" ]; then - printf "Error: Number of pods in running state is less than total ips in the azure ednpoint file" >&2 + printf "Error: Number of pods in running state is less than total ips in the azure endpoint file" >&2 exit 1 fi @@ -92,7 +106,25 @@ do fi done + num_of_cns_endpoints=$(cat cns_endpoints.json | jq -r '[.IPConfigurationStatus | .[] | select(.IPAddress != null)] | length') + cns_endpoint_ips=$(cat cns_endpoints.json | jq -r '(.IPConfigurationStatus | .[] | select(.IPAddress != null) | .IPAddress)') + echo "Number of cns endpoints: $num_of_cns_endpoints" + + if [ "$num_of_pod_ips" != "$num_of_cns_endpoints" ]; then + printf "Error: Number of pods in running state is less than total ips in the cns endpoint file" >&2 + exit 1 + fi + + for ip in "${pod_ips[@]}" + do + find_in_array "$cns_endpoint_ips" "$ip" "cns_endpoints.json" + if [[ $? -eq 1 ]]; then + printf "Error: %s Not found in the cns_endpoints.json" "$ip" >&2 + exit 1 + fi + done + #We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706 kubectl exec -i "$privileged_pod" -n kube-system -- bash -c "chroot /host /bin/bash -c 'systemctl restart systemd-networkd'" - rm -rf cilium_endpoints.json azure_endpoints.json + rm -rf cilium_endpoints.json azure_endpoints.json cns_endpoints.json done diff --git a/test/integration/manifests/cilium/cilium-agent/clusterrole.yaml b/test/integration/manifests/cilium/cilium-agent/clusterrole.yaml new file mode 100644 index 0000000000..881ec2255e --- /dev/null +++ b/test/integration/manifests/cilium/cilium-agent/clusterrole.yaml @@ -0,0 +1,95 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumbgploadbalancerippools + - ciliumbgppeeringpolicies + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumegressnatpolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + - ciliumendpoints/status + - ciliumendpoints + verbs: + - patch diff --git a/test/integration/manifests/cilium/cilium-agent/clusterrolebinding.yaml b/test/integration/manifests/cilium/cilium-agent/clusterrolebinding.yaml new file mode 100644 index 0000000000..f5d39b0ffd --- /dev/null +++ b/test/integration/manifests/cilium/cilium-agent/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system diff --git a/test/integration/manifests/cilium/cilium-agent/daemonset.yaml b/test/integration/manifests/cilium/cilium-agent/daemonset.yaml new file mode 100644 index 0000000000..b4fd32ab45 --- /dev/null +++ b/test/integration/manifests/cilium/cilium-agent/daemonset.yaml @@ -0,0 +1,417 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/managed-by: Helm + k8s-app: cilium + name: cilium + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: cilium + template: + metadata: + annotations: + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined + container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined + container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + k8s-app: cilium + kubernetes.azure.com/ebpf-dataplane: cilium + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + containers: + - args: + - --config-dir=/tmp/cilium/config-map + command: + - cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: CILIUM_CUSTOM_CNI_CONF + value: "true" + image: mcr.microsoft.com/oss/cilium/cilium:1.12.5.1 + imagePullPolicy: IfNotPresent + lifecycle: + postStart: + exec: + command: + - /cni-install.sh + preStop: + exec: + command: + - /cni-uninstall.sh + livenessProbe: + failureThreshold: 10 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + name: cilium-agent + ports: + - containerPort: 9962 + hostPort: 9962 + name: prometheus + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + resources: {} + securityContext: + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + startupProbe: + failureThreshold: 105 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /host/opt/cni/bin + name: cni-path + - mountPath: /host/etc/cni/net.d + name: etc-cni-netd + - mountPath: /var/lib/cilium/clustermesh + name: clustermesh-secrets + readOnly: true + - mountPath: /tmp/cilium/config-map + name: cilium-config-path + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + dnsPolicy: ClusterFirst + hostNetwork: true + initContainers: + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + image: mcr.microsoft.com/oss/cilium/cilium:1.12.5.1 + imagePullPolicy: IfNotPresent + name: mount-cgroup + resources: {} + securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + env: + - name: BIN_PATH + value: /opt/cni/bin + image: mcr.microsoft.com/oss/cilium/cilium:1.12.5.1 + imagePullPolicy: IfNotPresent + name: apply-sysctl-overwrites + resources: {} + securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: mcr.microsoft.com/oss/cilium/cilium:1.12.5.1 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps + - command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-state + name: cilium-config + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-bpf-state + name: cilium-config + optional: true + image: mcr.microsoft.com/oss/cilium/cilium:1.12.5.1 + imagePullPolicy: IfNotPresent + name: clean-cilium-state + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + name: cilium-cgroup + - mountPath: /var/run/cilium + name: cilium-run + - command: + - bash + - -cex + - | + export LD_LIBRARY_PATH=/host/lib/systemd:/host/usr/lib/aarch64-linux-gnu:/host/usr/lib/x86_64-linux-gnu + export SYSTEMD_VERSION="$(/host/lib/systemd/systemd --version | head -n 1 | cut -d' ' -f2)" + [[ $SYSTEMD_VERSION -ge 249 ]] && { + mkdir -p /host/etc/systemd/networkd.conf.d + echo -e "[Network]\nManageForeignRoutes=no\nManageForeignRoutingPolicyRules=no\n" \ + >/host/etc/systemd/networkd.conf.d/99-cilium-foreign-routes.conf + chmod -R u+rwX,go+rX /host/etc/systemd/networkd.conf.d + } || exit 0 + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + imagePullPolicy: IfNotPresent + name: systemd-networkd-overrides + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/etc/systemd + name: host-etc-systemd + - mountPath: /host/lib/systemd + name: host-lib-systemd + readOnly: true + - mountPath: /host/usr/lib + name: host-usr-lib + readOnly: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: cilium + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /etc/systemd + type: DirectoryOrCreate + name: host-etc-systemd + - hostPath: + path: /lib/systemd + type: DirectoryOrCreate + name: host-lib-systemd + - hostPath: + path: /usr/lib + type: DirectoryOrCreate + name: host-usr-lib + - hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + name: cilium-run + - hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + name: bpf-maps + - hostPath: + path: /proc + type: Directory + name: hostproc + - hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + name: cilium-cgroup + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-path + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: etc-cni-netd + - hostPath: + path: /lib/modules + type: "" + name: lib-modules + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock + - name: clustermesh-secrets + secret: + defaultMode: 256 + optional: true + secretName: cilium-clustermesh + - configMap: + defaultMode: 420 + name: cilium-config + name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 2 + type: RollingUpdate diff --git a/test/integration/manifests/cilium/cilium-agent/serviceaccount.yaml b/test/integration/manifests/cilium/cilium-agent/serviceaccount.yaml new file mode 100644 index 0000000000..edf2e96e34 --- /dev/null +++ b/test/integration/manifests/cilium/cilium-agent/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system + diff --git a/test/integration/manifests/cilium/cilium-config.yaml b/test/integration/manifests/cilium/cilium-config.yaml new file mode 100644 index 0000000000..cb53484746 --- /dev/null +++ b/test/integration/manifests/cilium/cilium-config.yaml @@ -0,0 +1,88 @@ +apiVersion: v1 +data: + agent-not-ready-taint-key: node.cilium.io/agent-not-ready + arping-refresh-period: 30s + auto-direct-node-routes: "false" + bpf-lb-external-clusterip: "false" + bpf-lb-map-max: "65536" + bpf-lb-mode: snat + bpf-map-dynamic-size-ratio: "0.0025" + bpf-policy-map-max: "16384" + bpf-root: /sys/fs/bpf + cgroup-root: /run/cilium/cgroupv2 + cilium-endpoint-gc-interval: 5m0s + cluster-id: "0" + cluster-name: default + debug: "false" + disable-cnp-status-updates: "true" + disable-endpoint-crd: "false" + enable-auto-protect-node-port-range: "true" + enable-bgp-control-plane: "false" + enable-bpf-clock-probe: "true" + enable-endpoint-health-checking: "false" + enable-endpoint-routes: "true" + enable-health-check-nodeport: "true" + enable-health-checking: "true" + enable-host-legacy-routing: "true" + enable-hubble: "false" + enable-ipv4: "true" + enable-ipv4-masquerade: "false" + enable-ipv6: "false" + enable-ipv6-masquerade: "false" + enable-k8s-terminating-endpoint: "true" + enable-l2-neigh-discovery: "true" + enable-l7-proxy: "false" + enable-local-node-route: "false" + enable-local-redirect-policy: "false" + enable-metrics: "true" + enable-policy: default + enable-remote-node-identity: "true" + enable-session-affinity: "true" + enable-svc-source-range-check: "true" + enable-vtep: "false" + enable-well-known-identities: "false" + enable-xt-socket-fallback: "true" + identity-allocation-mode: crd + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + ipam: delegated-plugin + kube-proxy-replacement: strict + kube-proxy-replacement-healthz-bind-address: "" + local-router-ipv4: 169.254.23.0 + metrics: +cilium_bpf_map_pressure + monitor-aggregation: medium + monitor-aggregation-flags: all + monitor-aggregation-interval: 5s + node-port-bind-protection: "true" + nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 + operator-prometheus-serve-addr: :9963 + preallocate-bpf-maps: "false" + procfs: /host/proc + prometheus-serve-addr: :9962 + remove-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + sidecar-istio-proxy-image: cilium/istio_proxy + synchronize-k8s-nodes: "true" + tofqdns-dns-reject-response-code: refused + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: 0s + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-response-max-delay: 100ms + tunnel: disabled + unmanaged-pod-watcher-interval: "15" + vtep-cidr: "" + vtep-endpoint: "" + vtep-mac: "" + vtep-mask: "" +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/managed-by: Helm + name: cilium-config + namespace: kube-system diff --git a/test/integration/manifests/cilium/cilium-operator/clusterrole.yaml b/test/integration/manifests/cilium/cilium-operator/clusterrole.yaml new file mode 100644 index 0000000000..a507d5e63f --- /dev/null +++ b/test/integration/manifests/cilium/cilium-operator/clusterrole.yaml @@ -0,0 +1,173 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + verbs: + - create + - update + - get + - list + - watch + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumbgploadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumegressnatpolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update diff --git a/test/integration/manifests/cilium/cilium-operator/clusterrolebinding.yaml b/test/integration/manifests/cilium/cilium-operator/clusterrolebinding.yaml new file mode 100644 index 0000000000..eb164361d4 --- /dev/null +++ b/test/integration/manifests/cilium/cilium-operator/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system diff --git a/test/integration/manifests/cilium/cilium-operator/deployment.yaml b/test/integration/manifests/cilium/cilium-operator/deployment.yaml new file mode 100644 index 0000000000..314e523523 --- /dev/null +++ b/test/integration/manifests/cilium/cilium-operator/deployment.yaml @@ -0,0 +1,117 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator +spec: + replicas: 1 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + kubernetes.azure.com/ebpf-dataplane: cilium + spec: + containers: + - name: cilium-operator + image: "mcr.microsoft.com/oss/cilium/operator-generic:1.12.5" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccount: "cilium-operator" + serviceAccountName: "cilium-operator" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config diff --git a/test/integration/manifests/cilium/cilium-operator/serviceaccount.yaml b/test/integration/manifests/cilium/cilium-operator/serviceaccount.yaml new file mode 100644 index 0000000000..be4bfc048a --- /dev/null +++ b/test/integration/manifests/cilium/cilium-operator/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system