Add tests and use updated pod killer (#85)

* Refactor API client creation

This helps us in writing unit tests for
the driver.

* update local setting for testing

* Run tests as part of build

Generate required mocks and run tests as part
of build.

* Output running test names and status

* WIP: expand test coverage

TODO: update quobyte api to proper version

* RBAC and CSI CRD update

also, includes some cleanup

* Log pod api requests in test cluster

Use audit logger to log k8s-api requests
in the test cluster.

* Update pod killer RBAC

* Fix pod killer scaling issue

Pod killer enumerates pods to identify
relevant pod (uuid to name/namespace resolution
is not possible) to locate pod on each k8s node.
Therefore is not scalable (overloads k8s api) to
large k8s clusters.

The new pod killer is split into pod killer controller
and mount point monitor. The controller which is deployed
on single node watches for k8s pods (does not enumerate all
the pods). The mounts monitors run on each k8s node and
report stale mounts to the controller, the controller
deletes pod(s) those have the stale mounts.

Since pods are watched (periodically synced) on single
node this should not cause overload on the k8s api server.

* Let dns be tried internally in k8s first

* Revert pod info on mount

* Fix pod killer cache definition

* Fix service acccount name

* revert dns

* Dynamically build pod killer cache URL

* Fine tune pod killer resources

Rename files to indicate resource used for
cache. Also, use deployment instead of stateful
sets for the pod killer cache, as cache does
not have any external state (required
state is pulled dynamically from k8s api).

* Use pod killer v0.2.0

* Update golang api to v1.3.0

* Adapt log collector to new pod killer

* Use pod killer container resource settings for backward compatibility

* Update template tests

* Update test config with recent sidecar containers

* Remove unused 2.x test configs

Quobyte CSI driver is no longer tested against
Quobyte 2.x versions, therefore remove the test
configs used for 2.x testing.

---------

Co-authored-by: venkat <venkat>

.gitignore

@@ -1,4 +1,4 @@
 .*
 src/quobyte-csi
 kind-cluster/*.pem
-kind-cluster/kind-csi-experiment-testing-corp/
+kind-cluster/kind-csi-testing/

csi-driver-templates/templates/csi-driver.yaml

@@ -1,3 +1,4 @@
 {{- include "quobyte-csi-driver.CSIDriverObject" . }}
 {{- include "quobyte-csi-driver.controllerPod" . }}
 {{- include "quobyte-csi-driver.nodeDriverPod" . }}
+{{- include "quobyte-csi-driver.podKiller.cachePod" . }}

csi-driver-templates/templates/pods/_quobyte_csi_controller_pod.tpl

@@ -19,7 +19,6 @@ spec:
   selector:
     matchLabels:
       app: quobyte-csi-controller-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
-  serviceName: quobyte-csi-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }} 
   replicas: {{ .Values.quobyte.csiControllerReplicas }}
   template:
     metadata:

csi-driver-templates/templates/pods/_quobyte_csi_node_driver_pod.tpl

@@ -29,7 +29,7 @@ spec:
       containers:
         {{- include "csi.sidecar.nodeRegistrarContainer" . | indent 8 }}
         {{- include "quobyte-csi-driver.nodeDriverContainer" . | indent 8 }}
-        {{- include "quobyte-csi-driver.podKillerContainer" . | indent 8 }}
+        {{- include "quobyte-csi-driver.podKiller.mountMonitor" . | indent 8 }}
       {{- include "quobyte-csi-driver.nodeDriverPodVolumeAttachments" . | indent 6 }}
 ---
 {{- end }}

csi-driver-templates/templates/pods/_quobyte_csi_pod_killer_cache.tpl

@@ -0,0 +1,47 @@
+{{- define "quobyte-csi-driver.podKiller.cachePod" }}
+{{- if .Values.quobyte.podKiller.enable }}
+---
+{{- include "quobyte-csi-driver.podKiller.cacheServiceAccount" . }}
+---
+{{- include "quobyte-csi-driver.podKiller.cacheRbac" . }}
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: quobyte-csi-pod-killer-cache-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: quobyte-csi-pod-killer-cache-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: quobyte-csi-pod-killer-cache-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+        role: quobyte-csi-pod-killer-cache-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+    spec:
+      priorityClassName: system-cluster-critical
+      serviceAccount: quobyte-csi-pod-killer-cache-sa-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+{{- if .Values.quobyte.tolerations }}
+      tolerations: 
+{{ toYaml .Values.quobyte.tolerations | indent 8 }}
+{{- end }}
+      containers:
+        {{- include "quobyte-csi-driver.podKiller.cacheContainer" . | indent 8 }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: quobyte-pod-killer-cache
+  namespace: kube-system
+spec:
+  selector:
+    app: quobyte-csi-pod-killer-cache-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 8080
+---
+{{- end }}
+{{- end }}

csi-driver-templates/templates/pods/containers/_quobyte_csi_controller_container.tpl

@@ -13,7 +13,7 @@
     allowPrivilegeEscalation: true
   image: {{ .Values.quobyte.dev.csiImage }}
   imagePullPolicy: "IfNotPresent"
-  args :
+  args:
     - "--csi_socket=$(CSI_ENDPOINT)"
     - "--quobyte_mount_path=$(QUOBYTE_MOUNT_PATH)"
     - "--node_name=$(KUBE_NODE_NAME)"

csi-driver-templates/templates/pods/containers/_quobyte_csi_pod_killer_cache_container.tpl

@@ -0,0 +1,16 @@
+{{- define "quobyte-csi-driver.podKiller.cacheContainer" }}
+- name: quobyte-csi-pod-killer-cache
+{{- if .Values.resources }}
+{{- if .Values.resources.podKillerContainer }}
+  resources: 
+{{ toYaml .Values.resources.podKillerContainer | indent 4 }}
+{{- end }}
+{{- end }}
+  image: {{ .Values.quobyte.dev.podKillerImage }}
+  ports:
+    - containerPort: 8080
+  imagePullPolicy: "IfNotPresent"
+  args:
+    - "--driver_name={{ .Values.quobyte.csiProvisionerName }}"
+    - "--role=cache"
+{{- end}}

csi-driver-templates/templates/pods/containers/_quobyte_pod_killer_container.tpl → csi-driver-templates/templates/pods/containers/_quobyte_pod_mount_monitor_container.tpl

@@ -1,6 +1,6 @@
-{{- define "quobyte-csi-driver.podKillerContainer" }}
+{{- define "quobyte-csi-driver.podKiller.mountMonitor" }}
 {{- if .Values.quobyte.podKiller.enable }}
-- name: quobyte-pod-killer
+- name: quobyte-csi-mount-monitor
 {{- if .Values.resources }}
 {{- if .Values.resources.podKillerContainer }}
   resources: 
@@ -11,11 +11,17 @@
     privileged: true
   image: {{ .Values.quobyte.dev.podKillerImage }}
   imagePullPolicy: "IfNotPresent"
-  args :
+  args:
     - "--node_name=$(KUBE_NODE_NAME)"
     - "--driver_name={{ .Values.quobyte.csiProvisionerName }}"
+    - "--service_url=http://quobyte-pod-killer-cache.$(NAMESPACE).svc.cluster.local:80/"
     - "--monitoring_interval={{ .Values.quobyte.podKiller.monitoringInterval }}"
+    - "--role=monitor"
   env:
+    - name: NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
     - name: NODE_ID
       valueFrom:
         fieldRef:

csi-driver-templates/templates/pods/rbac/_quobyte_csi_pod_killer_cache_rbac.tpl

@@ -0,0 +1,30 @@
+{{- define "quobyte-csi-driver.podKiller.cacheRbac" }}
+{{- if .Values.quobyte.podKiller.enable }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: quobyte-csi-pod-killer-cache-role-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["list", "watch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: quobyte-csi-pod-killer-cache-binding-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+subjects:
+  - kind: ServiceAccount
+    name: quobyte-csi-pod-killer-cache-sa-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: quobyte-csi-pod-killer-cache-role-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+  apiGroup: rbac.authorization.k8s.io
+---
+{{- end }}
+{{- end }}

csi-driver-templates/templates/pods/rbac/_sidecar_node_registrar_rbac.tpl

@@ -11,10 +11,7 @@ rules:
   {{- if .Values.quobyte.podKiller.enable }}
   - apiGroups: [""]
     resources: ["pods"]
-    verbs: ["list", "delete"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["list"]
+    verbs: ["delete"]
   {{- end }}
   {{- if .Values.quobyte.podSecurityPolicies }} 
   - apiGroups: ['policy']

csi-driver-templates/templates/pods/serviceAccounts/_quobyte_csi_pod_killer_cache_service_account.tpl

@@ -0,0 +1,9 @@
+{{- define "quobyte-csi-driver.podKiller.cacheServiceAccount"}}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: quobyte-csi-pod-killer-cache-sa-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
+  namespace: kube-system
+---
+{{- end}}