From 167bfae7febff5d4f64ac8483f4ea5dfd1f9ffe2 Mon Sep 17 00:00:00 2001
From: Curt Bushko <cbushko@gmail.com>
Date: Mon, 27 Jun 2022 10:19:17 -0400
Subject: [PATCH] CNI Helm Charts

Add helm charts for CNI installer
---
 charts/consul/templates/cni-clusterrole.yaml  |  23 ++
 .../templates/cni-clusterrolebinding.yaml     |  20 ++
 charts/consul/templates/cni-daemonset.yaml    |  85 +++++
 .../templates/cni-podsecuritypolicy.yaml      |  31 ++
 .../consul/templates/cni-resourcequota.yaml   |  22 ++
 .../consul/templates/cni-serviceaccount.yaml  |  19 ++
 .../templates/connect-inject-clusterrole.yaml |  10 +-
 .../templates/connect-inject-deployment.yaml  |   1 +
 charts/consul/test/unit/cni-clusterrole.bats  |  31 ++
 .../test/unit/cni-clusterrolebinding.bats     |  57 ++++
 charts/consul/test/unit/cni-daemonset.bats    | 302 ++++++++++++++++++
 .../test/unit/cni-podsecuritypolicy.bats      |  32 ++
 .../consul/test/unit/cni-resourcequota.bats   |  57 ++++
 .../consul/test/unit/cni-serviceaccount.bats  |  53 +++
 .../test/unit/connect-inject-clusterrole.bats |  38 ++-
 .../test/unit/connect-inject-deployment.bats  |  30 ++
 charts/consul/values.yaml                     |  79 ++++-
 .../testdata/ZZZ-consul-cni-kubeconfig.golden |   1 +
 18 files changed, 882 insertions(+), 9 deletions(-)
 create mode 100644 charts/consul/templates/cni-clusterrole.yaml
 create mode 100644 charts/consul/templates/cni-clusterrolebinding.yaml
 create mode 100644 charts/consul/templates/cni-daemonset.yaml
 create mode 100644 charts/consul/templates/cni-podsecuritypolicy.yaml
 create mode 100644 charts/consul/templates/cni-resourcequota.yaml
 create mode 100644 charts/consul/templates/cni-serviceaccount.yaml
 create mode 100644 charts/consul/test/unit/cni-clusterrole.bats
 create mode 100644 charts/consul/test/unit/cni-clusterrolebinding.bats
 create mode 100644 charts/consul/test/unit/cni-daemonset.bats
 create mode 100644 charts/consul/test/unit/cni-podsecuritypolicy.bats
 create mode 100644 charts/consul/test/unit/cni-resourcequota.bats
 create mode 100644 charts/consul/test/unit/cni-serviceaccount.bats

diff --git a/charts/consul/templates/cni-clusterrole.yaml b/charts/consul/templates/cni-clusterrole.yaml
new file mode 100644
index 0000000000..744a4af864
--- /dev/null
+++ b/charts/consul/templates/cni-clusterrole.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.connectInject.cni.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "consul.fullname" . }}-cni
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: cni 
+rules:
+- apiGroups: [""]
+  resources: 
+  - pods
+  verbs: 
+  - get
+  - list
+  - watch
+  - patch
+  - update
+{{- end }}
diff --git a/charts/consul/templates/cni-clusterrolebinding.yaml b/charts/consul/templates/cni-clusterrolebinding.yaml
new file mode 100644
index 0000000000..86c19d86aa
--- /dev/null
+++ b/charts/consul/templates/cni-clusterrolebinding.yaml
@@ -0,0 +1,20 @@
+{{- if .Values.connectInject.cni.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ template "consul.fullname" . }}-cni
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: cni
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "consul.fullname" . }}-cni
+subjects:
+- kind: ServiceAccount
+  name: {{ template "consul.fullname" . }}-cni
+  namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/charts/consul/templates/cni-daemonset.yaml b/charts/consul/templates/cni-daemonset.yaml
new file mode 100644
index 0000000000..010d0c8e17
--- /dev/null
+++ b/charts/consul/templates/cni-daemonset.yaml
@@ -0,0 +1,85 @@
+{{- if (and (.Values.connectInject.cni.enabled) (not .Values.connectInject.enabled)) }}{{ fail "connectInject.enabled must be true if connectInject.cni.enabled is true" }}{{ end -}}
+{{- if .Values.connectInject.cni.enabled }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ template "consul.fullname" . }}-cni
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: cni 
+spec:
+  {{- if .Values.connectInject.cni.updateStrategy }}
+  updateStrategy:
+    {{ tpl .Values.connectInject.cni.updateStrategy . | nindent 4 | trim }}
+  {{- end }}
+  selector:
+    matchLabels:
+      app: {{ template "consul.name" . }}
+      chart: {{ template "consul.chart" . }}
+      release: {{ .Release.Name }}
+      component: cni
+  template:
+    metadata:
+      labels:
+        app: {{ template "consul.name" . }}
+        chart: {{ template "consul.chart" . }}
+        release: {{ .Release.Name }}
+        component: cni
+      annotations:
+        consul.hashicorp.com/connect-inject: "false"
+    spec:
+      # consul-cni only runs on linux operating systems
+      nodeSelector:
+        kubernetes.io/os: linux
+      tolerations:
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      # Tell kubernetes that this daemonset is critical so that it will be scheduled on a new node before other pods
+      priorityClassName: system-node-critical
+      serviceAccountName: {{ template "consul.fullname" . }}-cni
+      {{- if not .Values.global.openshift.enabled }}
+      securityContext:
+        {{- toYaml .Values.connectInject.cni.securityContext | nindent 8 -}}
+      {{- end }}
+      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
+      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
+      terminationGracePeriodSeconds: 10
+      containers:
+        # This container installs the consul CNI binaries and CNI network config file on each node
+        - name: install-cni
+          image: {{ .Values.global.imageK8S }}
+          securityContext:
+            privileged: true
+          command:
+            - consul-k8s-control-plane
+            - install-cni
+            - -multus={{ .Values.connectInject.cni.multus }}
+            - -log-level={{ default .Values.global.logLevel .Values.connectInject.cni.logLevel }}
+            - -cni-bin-dir={{ .Values.connectInject.cni.cniBinDir }}
+            - -cni-net-dir={{ .Values.connectInject.cni.cniNetDir }}
+            - -dns-prefix={{ template "consul.fullname" . }}
+          {{- with .Values.connectInject.cni.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - mountPath: {{ .Values.connectInject.cni.cniBinDir }}
+              name: cni-bin-dir
+            - mountPath: {{ .Values.connectInject.cni.cniNetDir }}
+              name: cni-net-dir
+      volumes:
+        # Used to install CNI.
+        - name: cni-bin-dir
+          hostPath:
+            path: {{ .Values.connectInject.cni.cniBinDir }} 
+        - name: cni-net-dir
+          hostPath:
+            path: {{ .Values.connectInject.cni.cniNetDir }} 
+{{- end }}
diff --git a/charts/consul/templates/cni-podsecuritypolicy.yaml b/charts/consul/templates/cni-podsecuritypolicy.yaml
new file mode 100644
index 0000000000..ddc75913ae
--- /dev/null
+++ b/charts/consul/templates/cni-podsecuritypolicy.yaml
@@ -0,0 +1,31 @@
+{{- if (and .Values.connectInject.cni.enabled .Values.global.enablePodSecurityPolicies) }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: {{ template "consul.fullname" . }}-cni
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: cni 
+spec:
+  privileged: true
+  # Required to prevent escalations to root.
+  allowPrivilegeEscalation: false
+  volumes:
+  - hostPath
+  - secret
+  - emptyDir
+  hostNetwork: false
+  readOnlyRootFilesystem: false
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+{{- end }}
diff --git a/charts/consul/templates/cni-resourcequota.yaml b/charts/consul/templates/cni-resourcequota.yaml
new file mode 100644
index 0000000000..abfe5a8876
--- /dev/null
+++ b/charts/consul/templates/cni-resourcequota.yaml
@@ -0,0 +1,22 @@
+{{- if .Values.connectInject.cni.enabled }}
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: {{ template "consul.fullname" . }}-cni
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: cni 
+spec:
+  hard:
+    pods: {{ .Values.connectInject.cni.resourceQuota.pods | quote }}
+  scopeSelector:
+    matchExpressions:
+    - operator: In
+      scopeName: PriorityClass
+      values:
+      - system-node-critical
+{{- end }}
diff --git a/charts/consul/templates/cni-serviceaccount.yaml b/charts/consul/templates/cni-serviceaccount.yaml
new file mode 100644
index 0000000000..6b2a7627f7
--- /dev/null
+++ b/charts/consul/templates/cni-serviceaccount.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.connectInject.cni.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "consul.fullname" . }}-cni
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: cni 
+{{- with .Values.global.imagePullSecrets }}
+imagePullSecrets:
+{{- range . }}
+- name: {{ .name }}
+{{- end }}
+{{- end }}
+{{- end}}
diff --git a/charts/consul/templates/connect-inject-clusterrole.yaml b/charts/consul/templates/connect-inject-clusterrole.yaml
index 846e70132a..565f146d02 100644
--- a/charts/consul/templates/connect-inject-clusterrole.yaml
+++ b/charts/consul/templates/connect-inject-clusterrole.yaml
@@ -19,11 +19,19 @@ rules:
   - get
 {{- end }}
 - apiGroups: [ "" ]
-  resources: [ "pods", "endpoints", "services", "namespaces", "nodes" ]
+  resources: [ "endpoints", "services", "namespaces" ]
   verbs:
   - "get"
   - "list"
   - "watch"
+- apiGroups: [ "" ]
+  resources:
+  - pods
+  verbs:
+  - "get"
+  - "list"
+  - "watch"
+  - "update"
 - apiGroups:
   - coordination.k8s.io
   resources:
diff --git a/charts/consul/templates/connect-inject-deployment.yaml b/charts/consul/templates/connect-inject-deployment.yaml
index 266eeb4994..c924b95acd 100644
--- a/charts/consul/templates/connect-inject-deployment.yaml
+++ b/charts/consul/templates/connect-inject-deployment.yaml
@@ -139,6 +139,7 @@ spec:
                 {{- else }}
                 -default-enable-transparent-proxy=false \
                 {{- end }}
+                -enable-cni={{ .Values.connectInject.cni.enabled }} \
                 {{- if .Values.global.peering.enabled }}
                 -enable-peering=true \
                 {{- if (eq .Values.global.peering.tokenGeneration.serverAddresses.source "") }}
diff --git a/charts/consul/test/unit/cni-clusterrole.bats b/charts/consul/test/unit/cni-clusterrole.bats
new file mode 100644
index 0000000000..02675ed882
--- /dev/null
+++ b/charts/consul/test/unit/cni-clusterrole.bats
@@ -0,0 +1,31 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "cni/ClusterRole: disabled by default" {
+  cd `chart_dir`
+  assert_empty helm template \
+      -s templates/cni-clusterrole.yaml  \
+      .
+}
+
+@test "cni/ClusterRole: enabled with connectInject.cni.enabled=true and connectInject.enabled=true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-clusterrole.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [[ "${actual}" == "true" ]]
+}
+
+@test "cni/ClusterRole: disabled with connectInject.cni.enabled=false and connectInject.enabled=true" {
+  cd `chart_dir`
+  assert_empty helm template \
+      --set 'connectInject.cni.enabled=false' \
+      --set 'connectInject.enabled=true' \
+      -s templates/cni-clusterrole.yaml  \
+      .
+}
+
diff --git a/charts/consul/test/unit/cni-clusterrolebinding.bats b/charts/consul/test/unit/cni-clusterrolebinding.bats
new file mode 100644
index 0000000000..ba217e7706
--- /dev/null
+++ b/charts/consul/test/unit/cni-clusterrolebinding.bats
@@ -0,0 +1,57 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "cni/ClusterRoleBinding: disabled by default" {
+  cd `chart_dir`
+  assert_empty helm template \
+      -s templates/cni-clusterrolebinding.yaml  \
+      .
+}
+
+@test "cni/ClusterRoleBinding: enabled with connectInject.cni.enabled=true and connectInject.enabled=true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-clusterrolebinding.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [[ "${actual}" == "true" ]]
+}
+
+@test "cni/ClusterRoleBinding: disabled with connectInject.cni.enabled=false and connectInject.enabled=true" {
+  cd `chart_dir`
+  assert_empty helm template \
+      --set 'connectInject.cni.enabled=false' \
+      --set 'connectInject.enabled=true' \
+      -s templates/cni-clusterrolebinding.yaml  \
+      .
+}
+
+#--------------------------------------------------------------------
+# subjects
+
+@test "cni/ClusterRoleBinding: subject name is correct" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-clusterrolebinding.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r '.subjects[0].name' | tee /dev/stderr)
+  [ "${actual}" = "release-name-consul-cni" ]
+}
+
+@test "cni/ClusterRoleBinding: subject namespace is correct" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-clusterrolebinding.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --namespace foo \
+      . | tee /dev/stderr |
+      yq -r '.subjects[0].namespace' | tee /dev/stderr)
+  [ "${actual}" = "foo" ]
+}
+
diff --git a/charts/consul/test/unit/cni-daemonset.bats b/charts/consul/test/unit/cni-daemonset.bats
new file mode 100644
index 0000000000..ebf63088eb
--- /dev/null
+++ b/charts/consul/test/unit/cni-daemonset.bats
@@ -0,0 +1,302 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "cni/DaemonSet: disabled by default" {
+  cd `chart_dir`
+  assert_empty helm template \
+      -s templates/cni-daemonset.yaml  \
+      .
+}
+
+@test "cni/DaemonSet: enabled with connectInject.cni.enabled=true and connectInject.enabled=true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [[ "${actual}" == "true" ]]
+}
+
+@test "cni/DaemonSet: disabled with connectInject.cni.enabled=false and connectInject.enabled=true" {
+  cd `chart_dir`
+  assert_empty helm template \
+      --set 'connectInject.cni.enabled=false' \
+      --set 'connectInject.enabled=true' \
+      -s templates/cni-daemonset.yaml  \
+      .
+}
+
+@test "cni/DaemonSet: throws error when connectInject.enabled=true and connectInject.enabled=false" {
+  cd `chart_dir`
+  run helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=false' \
+      -s templates/cni-daemonset.yaml  \
+      .
+
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "connectInject.enabled must be true if connectInject.cni.enabled is true" ]]
+}
+
+@test "cni/DaemonSet: image defaults to global.imageK8S" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.imageK8S=foo' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].image' | tee /dev/stderr)
+  [ "${actual}" = "foo" ]
+}
+
+@test "cni/Daemonset: all command arguments" {
+  cd `chart_dir`
+  local cmd=$(helm template \
+      -s templates/cni-daemonset.yaml \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'connectInject.cni.multus=foo' \
+      --set 'connectInject.cni.logLevel=bar' \
+      --set 'connectInject.cni.cniBinDir=baz' \
+      --set 'connectInject.cni.cniNetDir=foo' \
+      bar \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("consul-k8s-control-plane"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("install-cni"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("multus=foo"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("log-level=bar"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("cni-bin-dir=baz"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("cni-net-dir=foo"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("dns-prefix=bar-consul"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+#--------------------------------------------------------------------
+# updateStrategy
+
+@test "cni/DaemonSet: no updateStrategy by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r '.spec.updateStrategy' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
+@test "cni/DaemonSet: updateStrategy can be set" {
+  cd `chart_dir`
+  local updateStrategy="type: RollingUpdate
+rollingUpdate:
+  maxUnavailable: 5
+"
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set "connectInject.cni.updateStrategy=${updateStrategy}" \
+      . | tee /dev/stderr | \
+      yq -c '.spec.updateStrategy == {"type":"RollingUpdate","rollingUpdate":{"maxUnavailable":5}}' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+
+#--------------------------------------------------------------------
+# resources
+
+@test "cni/DaemonSet: resources defined by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -rc '.spec.template.spec.containers[0].resources' | tee /dev/stderr)
+  [ "${actual}" = '{"limits":{"cpu":"50m","memory":"50Mi"},"requests":{"cpu":"50m","memory":"50Mi"}}' ]
+}
+
+@test "cni/DaemonSet: resources can be overridden" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'connectInject.cni.resources.foo=bar' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].resources.foo' | tee /dev/stderr)
+  [ "${actual}" = "bar" ]
+}
+
+#--------------------------------------------------------------------
+# securityContext
+
+@test "cni/DaemonSet: securityContext is not set when global.openshift.enabled=true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.openshift.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.securityContext' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
+@test "cni/DaemonSet: sets default security context settings" {
+  cd `chart_dir`
+  local security_context=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.securityContext' | tee /dev/stderr)
+
+  local actual=$(echo $security_context | jq -r .runAsNonRoot)
+  [ "${actual}" = "false" ]
+
+  local actual=$(echo $security_context | jq -r .runAsUser)
+  [ "${actual}" = "0" ]
+
+  local actual=$(echo $security_context | jq -r .runAsGroup)
+  [ "${actual}" = "0" ]
+}
+
+@test "cni/DaemonSet: can overwrite security context settings" {
+  cd `chart_dir`
+  local security_context=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'connectInject.cni.securityContext.runAsNonRoot=true' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.securityContext' | tee /dev/stderr)
+
+  local actual=$(echo $security_context | jq -r .runAsNonRoot)
+  [ "${actual}" = "true" ]
+}
+
+#--------------------------------------------------------------------
+# volumes
+
+@test "cni/DaemonSet: sets default cni-bin-dir volume hostPath" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r -c '.spec.template.spec.volumes[] | select(.name == "cni-bin-dir")' | tee /dev/stderr)
+      [ "${actual}" = '{"name":"cni-bin-dir","hostPath":{"path":"/opt/cni/bin"}}' ]
+}
+
+@test "cni/DaemonSet: sets default cni-net-dir volume hostPath" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r -c '.spec.template.spec.volumes[] | select(.name == "cni-net-dir")' | tee /dev/stderr)
+      [ "${actual}" = '{"name":"cni-net-dir","hostPath":{"path":"/etc/cni/net.d"}}' ]
+}
+
+@test "cni/DaemonSet: can overwrite default cni-bin-dir volume hostPath" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'connectInject.cni.cniBinDir=foo' \
+      . | tee /dev/stderr |
+      yq -r -c '.spec.template.spec.volumes[] | select(.name == "cni-bin-dir")' | tee /dev/stderr)
+      [ "${actual}" = '{"name":"cni-bin-dir","hostPath":{"path":"foo"}}' ]
+}
+
+@test "cni/DaemonSet: can overwrite cni-net-dir volume hostPath" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'connectInject.cni.cniNetDir=bar' \
+      . | tee /dev/stderr |
+      yq -r -c '.spec.template.spec.volumes[] | select(.name == "cni-net-dir")' | tee /dev/stderr)
+      [ "${actual}" = '{"name":"cni-net-dir","hostPath":{"path":"bar"}}' ]
+}
+
+#--------------------------------------------------------------------
+# volumeMounts
+
+@test "cni/DaemonSet: sets default host cni-bin-dir volumeMount" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r -c '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "cni-bin-dir")' | tee /dev/stderr)
+      [ "${actual}" = '{"mountPath":"/opt/cni/bin","name":"cni-bin-dir"}' ]
+}
+
+@test "cni/DaemonSet: sets default host cni-net-dir volumeMount" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r -c '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "cni-net-dir")' | tee /dev/stderr)
+      [ "${actual}" = '{"mountPath":"/etc/cni/net.d","name":"cni-net-dir"}' ]
+}
+
+@test "cni/DaemonSet: can overwrite host cni-bin-dir volumeMount" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'connectInject.cni.cniBinDir=foo' \
+      . | tee /dev/stderr |
+      yq -r -c '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "cni-bin-dir")' | tee /dev/stderr)
+      [ "${actual}" = '{"mountPath":"foo","name":"cni-bin-dir"}' ]
+}
+
+@test "cni/DaemonSet: can overwrite host cni-net-dir volumeMount" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-daemonset.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'connectInject.cni.cniNetDir=bar' \
+      . | tee /dev/stderr |
+      yq -r -c '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "cni-net-dir")' | tee /dev/stderr)
+      [ "${actual}" = '{"mountPath":"bar","name":"cni-net-dir"}' ]
+}
+
diff --git a/charts/consul/test/unit/cni-podsecuritypolicy.bats b/charts/consul/test/unit/cni-podsecuritypolicy.bats
new file mode 100644
index 0000000000..37df761995
--- /dev/null
+++ b/charts/consul/test/unit/cni-podsecuritypolicy.bats
@@ -0,0 +1,32 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "cni/PodSecurityPolicy: disabled by default" {
+  cd `chart_dir`
+  assert_empty helm template \
+      -s templates/cni-podsecuritypolicies.yaml  \
+      .
+}
+
+@test "cni/PodSecurityPolicy: disabled with cni disabled and global.enablePodSecurityPolicies=true" {
+  cd `chart_dir`
+  assert_empty helm template \
+      -s templates/cni-podsecuritypolicy.yaml  \
+      --set 'connectInject.cni.enabled=false' \
+      --set 'global.enablePodSecurityPolicies=true' \
+      .
+}
+
+@test "cni/PodSecurityPolicy: enabled with connectInject.cni.enabled=true and global.enablePodSecurityPolicies=true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-podsecuritypolicy.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.enablePodSecurityPolicies=true' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [[ "${actual}" == "true" ]]
+}
+
diff --git a/charts/consul/test/unit/cni-resourcequota.bats b/charts/consul/test/unit/cni-resourcequota.bats
new file mode 100644
index 0000000000..36c7a26b30
--- /dev/null
+++ b/charts/consul/test/unit/cni-resourcequota.bats
@@ -0,0 +1,57 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "cni/ResourceQuota: disabled by default" {
+  cd `chart_dir`
+  assert_empty helm template \
+      -s templates/cni-resourcequota.yaml  \
+      .
+}
+
+@test "cni/ResourceQuota: enabled with connectInject.cni.enabled=true and connectInject.enabled=true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-resourcequota.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [[ "${actual}" == "true" ]]
+}
+
+@test "cni/ResourceQuota: disabled with connectInject.cni.enabled=false and connectInject.enabled=true" {
+  cd `chart_dir`
+  assert_empty helm template \
+      --set 'connectInject.cni.enabled=false' \
+      --set 'connectInject.enabled=true' \
+      -s templates/cni-resourcequota.yaml  \
+      .
+}
+
+#--------------------------------------------------------------------
+# pods 
+
+@test "cni/ResourceQuota: resources defined by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-resourcequota.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -rc '.spec.hard.pods' | tee /dev/stderr)
+  [ "${actual}" = "5000" ]
+}
+
+@test "cni/ResourceQuota: resources can be overridden" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-resourcequota.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'connectInject.cni.resourceQuota.pods=bar' \
+      . | tee /dev/stderr |
+      yq -rc '.spec.hard.pods' | tee /dev/stderr)
+  [ "${actual}" = "bar" ]
+}
+
diff --git a/charts/consul/test/unit/cni-serviceaccount.bats b/charts/consul/test/unit/cni-serviceaccount.bats
new file mode 100644
index 0000000000..4f2071f823
--- /dev/null
+++ b/charts/consul/test/unit/cni-serviceaccount.bats
@@ -0,0 +1,53 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "cni/ServiceAccount: disabled by default" {
+  cd `chart_dir`
+  assert_empty helm template \
+      -s templates/cni-serviceaccount.yaml  \
+      .
+}
+
+@test "cni/ServiceAccount: enabled with connectInject.cni.enabled=true and connectInject.enabled=true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/cni-serviceaccount.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [[ "${actual}" == "true" ]]
+}
+
+@test "cni/ServiceAccount: disabled with connectInject.cni.enabled=false and connectInject.enabled=true" {
+  cd `chart_dir`
+  assert_empty helm template \
+      --set 'connectInject.cni.enabled=false' \
+      --set 'connectInject.enabled=true' \
+      -s templates/cni-serviceaccount.yaml  \
+      .
+}
+
+#--------------------------------------------------------------------
+# global.imagePullSecrets
+
+@test "cni/ServiceAccount: can set image pull secrets" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/cni-serviceaccount.yaml  \
+      --set 'connectInject.cni.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.imagePullSecrets[0].name=my-secret' \
+      --set 'global.imagePullSecrets[1].name=my-secret2' \
+      . | tee /dev/stderr)
+
+  local actual=$(echo "$object" |
+      yq -r '.imagePullSecrets[0].name' | tee /dev/stderr)
+  [ "${actual}" = "my-secret" ]
+
+  local actual=$(echo "$object" |
+      yq -r '.imagePullSecrets[1].name' | tee /dev/stderr)
+  [ "${actual}" = "my-secret2" ]
+}
+
diff --git a/charts/consul/test/unit/connect-inject-clusterrole.bats b/charts/consul/test/unit/connect-inject-clusterrole.bats
index 7939755f81..1b7d447a77 100644
--- a/charts/consul/test/unit/connect-inject-clusterrole.bats
+++ b/charts/consul/test/unit/connect-inject-clusterrole.bats
@@ -32,7 +32,7 @@ load _helpers
 #--------------------------------------------------------------------
 # rules
 
-@test "connectInject/ClusterRole: sets get, list, and watch access to pods, endpoints, services, and namespaces in all api groups" {
+@test "connectInject/ClusterRole: sets get, list, and watch access to endpoints, services, and namespaces in all api groups" {
   cd `chart_dir`
   local object=$(helm template \
       -s templates/connect-inject-clusterrole.yaml  \
@@ -42,9 +42,6 @@ load _helpers
       . | tee /dev/stderr |
       yq -r '.rules[0]' | tee /dev/stderr)
 
-  local actual=$(echo $object | yq -r '.resources[| index("pods")' | tee /dev/stderr)
-  [ "${actual}" != null ]
-
   local actual=$(echo $object | yq -r '.resources[| index("endpoints")' | tee /dev/stderr)
   [ "${actual}" != null ]
 
@@ -67,7 +64,7 @@ load _helpers
   [ "${actual}" != null ]
 }
 
-@test "connectInject/ClusterRole: sets create, get, list, and update access to leases in the coordination.k8s.io api group" {
+@test "connectInject/ClusterRole: sets get, list, watch and update access to pods in all api groups" {
   cd `chart_dir`
   local object=$(helm template \
       -s templates/connect-inject-clusterrole.yaml  \
@@ -77,6 +74,35 @@ load _helpers
       . | tee /dev/stderr |
       yq -r '.rules[1]' | tee /dev/stderr)
 
+  local actual=$(echo $object | yq -r '.resources[| index("pods")' | tee /dev/stderr)
+  [ "${actual}" != null ]
+
+  local actual=$(echo $object | yq -r '.apiGroups[0]' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+
+  local actual=$(echo $object | yq -r '.verbs | index("get")' | tee /dev/stderr)
+  [ "${actual}" != null ]
+
+  local actual=$(echo $object | yq -r '.verbs | index("list")' | tee /dev/stderr)
+  [ "${actual}" != null ]
+
+  local actual=$(echo $object | yq -r '.verbs | index("watch")' | tee /dev/stderr)
+  [ "${actual}" != null ]
+
+  local actual=$(echo $object | yq -r '.verbs | index("update")' | tee /dev/stderr)
+  [ "${actual}" != null ]
+}
+
+@test "connectInject/ClusterRole: sets create, get, list, and update access to leases in the coordination.k8s.io api group" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/connect-inject-clusterrole.yaml  \
+      --set 'global.enabled=false' \
+      --set 'client.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r '.rules[2]' | tee /dev/stderr)
+
   local actual=$(echo $object | yq -r '.resources[| index("leases")' | tee /dev/stderr)
   [ "${actual}" != null ]
 
@@ -166,7 +192,7 @@ load _helpers
       --set 'global.secretsBackend.vault.consulServerRole=bar' \
       --set 'global.secretsBackend.vault.consulCARole=test2' \
       . | tee /dev/stderr |
-      yq -r '.rules[2]' | tee /dev/stderr)
+      yq -r '.rules[3]' | tee /dev/stderr)
 
   local actual=$(echo $object | yq -r '.resources[0]' | tee /dev/stderr)
   [ "${actual}" = "mutatingwebhookconfigurations" ]
diff --git a/charts/consul/test/unit/connect-inject-deployment.bats b/charts/consul/test/unit/connect-inject-deployment.bats
index 52552c6721..95d0958564 100755
--- a/charts/consul/test/unit/connect-inject-deployment.bats
+++ b/charts/consul/test/unit/connect-inject-deployment.bats
@@ -1795,6 +1795,36 @@ EOF
   [ "${actual}" = "true" ]
 }
 
+#--------------------------------------------------------------------
+# cni 
+
+@test "connectInject/Deployment: cni is disabled by default" {
+  cd `chart_dir`
+  local cmd=$(helm template \
+      -s templates/connect-inject-deployment.yaml  \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-enable-cni=false"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "connectInject/Deployment: cni can be enabled by setting connectInject.cni.enabled=true" {
+  cd `chart_dir`
+  local cmd=$(helm template \
+      -s templates/connect-inject-deployment.yaml  \
+      --set 'connectInject.enabled=true' \
+      --set 'connectInject.cni.enabled=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-enable-cni=true"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 #--------------------------------------------------------------------
 # peering
 
diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
index dd38152c80..615e3bf1d8 100644
--- a/charts/consul/values.yaml
+++ b/charts/consul/values.yaml
@@ -550,8 +550,7 @@ global:
     # `<helm-release-name>-consul-federation`.
     createFederationSecret: false
 
-    # The name of the primary datacenter.  This should only be set for datacenters
-    # that are not the primary datacenter.
+    # The name of the primary datacenter.
     # @type: string
     primaryDatacenter: null
 
@@ -1932,6 +1931,82 @@ connectInject:
     # @type: integer
     maxUnavailable: null
 
+  # Configures consul-cni plugin for Consul Service mesh services
+  cni:
+    # If true, then all traffic redirection setup will use the consul-cni plugin. 
+    # Requires connectInject.enabled to also be true.
+    # @type: boolean
+    enabled: false
+ 
+    # If multus plugin is enabled or not. If true, the CNI installer will generate a consul CNI configuration
+    # in a location for the multus program to use. Also, the helm chart will create a custom resource for
+    # NetworkAttachmentDefintion that multus will consume. The values for cniBinDir and cniNetDir must also be 
+    # changed for multus to work. 
+    # Please refer to your cloud platform for more informaton on using multus.
+    # @type: boolean
+    multus: false
+
+    # Log level for the installer and plugin. Overrides global.logLevel
+    # @type: string
+    logLevel: null 
+    
+    # Location on the kubernetes node where the CNI plugin is installed. Shoud be the absolute path and start with a '/'
+    # Example on GKE:
+    #
+    # ```yaml
+    # cniBinDir: "/home/kubernetes/bin"
+    # ```
+    # @type: string
+    cniBinDir: "/opt/cni/bin"
+
+    # Location on the kubernetes node of all CNI configuration. Should be the absolute path and start with a '/'
+    # @type: string
+    cniNetDir: "/etc/cni/net.d"
+ 
+    # The resource settings for CNI installer daemonset.
+    # @recurse: false
+    # @type: map
+    resources:
+      requests:
+        memory: "50Mi"
+        cpu: "50m"
+      limits:
+        memory: "50Mi"
+        cpu: "50m"
+
+    # Resource quotas for running the daemonset as system critical pods
+    resourceQuota:
+      pods: 5000
+
+    # The security context for the CNI installer daemonset. This should be a YAML map corresponding to a
+    # Kubernetes [SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) object.
+    # By default, servers will run as root, with user ID `0` and group ID `0`.
+    # Note: if running on OpenShift, this setting is ignored because the user and group are set automatically
+    # by the OpenShift platform.
+    # @type: map
+    # @recurse: false
+    securityContext:
+      runAsNonRoot: false
+      runAsGroup: 0
+      runAsUser: 0
+
+    # updateStrategy for the CNI installer DaemonSet.
+    # See https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy.
+    # This should be a multi-line string mapping directly to the updateStrategy
+    #
+    # Example:
+    #
+    # ```yaml
+    # updateStrategy: |
+    #   rollingUpdate:
+    #     maxUnavailable: 5
+    #   type: RollingUpdate
+    # ```
+    #
+    # @type: string
+    updateStrategy: null
+
+
   # Configures metrics for Consul Connect services. All values are overridable
   # via annotations on a per-pod basis.
   metrics:
diff --git a/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden b/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden
index 0bf481f86f..ef76b8b48b 100644
--- a/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden
+++ b/control-plane/subcommand/install-cni/testdata/ZZZ-consul-cni-kubeconfig.golden
@@ -15,4 +15,5 @@ preferences: {}
 users:
 - name: consul-cni
   user:
+    as-user-extra: null
     token: eyJhbGciOiJSUzI1NiIsImtp