Fine tune pod killer resources

Rename files to indicate resource used for
cache. Also, use deployment instead of stateful
sets for the pod killer cache, as cache does
not have any external state (required
state is pulled dynamically from k8s api).

.gitignore

@@ -1,4 +1,4 @@
 .*
 src/quobyte-csi
 kind-cluster/*.pem
-kind-cluster/kind-csi-experiment-testing-corp/
+kind-cluster/kind-csi-testing/

csi-driver-templates/templates/pods/_quobyte_csi_controller_pod.tpl

@@ -19,7 +19,6 @@ spec:
   selector:
     matchLabels:
       app: quobyte-csi-controller-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
-  serviceName: quobyte-csi-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }} 
   replicas: {{ .Values.quobyte.csiControllerReplicas }}
   template:
     metadata:

csi-driver-templates/templates/pods/_quobyte_csi_pod_killer_controller.tpl → csi-driver-templates/templates/pods/_quobyte_csi_pod_killer_cache.tpl

@@ -5,7 +5,7 @@
 ---
 {{- include "quobyte-csi-driver.podKiller.cacheRbac" . }}
 ---
-kind: StatefulSet
+kind: Deployment
 apiVersion: apps/v1
 metadata:
   name: quobyte-csi-pod-killer-cache-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
@@ -14,7 +14,6 @@ spec:
   selector:
     matchLabels:
       app: quobyte-csi-pod-killer-cache-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }}
-  serviceName: quobyte-csi-pod-killer-cache-{{ .Values.quobyte.csiProvisionerName | replace "." "-"  }} 
   replicas: 1
   template:
     metadata:

kind-cluster/cleanup

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-CLUSTER_NAME='quobyte-csi-experiment-testing'
-TEST_CLUSTER_DIR='kind-cluster/kind-csi-experiment-testing-corp'
+CLUSTER_NAME='quobyte-csi-testing'
+TEST_CLUSTER_DIR='kind-cluster/kind-csi-testing'
 
 echo ''
 echo 'deleting kind images from docker'

kind-cluster/run_test

@@ -4,8 +4,8 @@ MOUNT_WITH_ACCESS_KEYS=${MOUNT_WITH_ACCESS_KEYS:-n}
 PATH="$(pwd):$PATH"
 CODE_BASE_DIR="$(pwd)"
 TEST_CASE_DIR="${TEST_CASE_DIR:-}"
-TEST_CLUSTER_DIR='kind-cluster/kind-csi-experiment-testing-corp'
-KIND_CLUSTER_NAME="quobyte-csi-experiment-testing"
+KIND_CLUSTER_NAME="quobyte-csi-testing"
+TEST_CLUSTER_DIR='kind-cluster/kind-csi-testing'
 CSI_PROVISIONER_NAME=${CSI_PROVISIONER_NAME:-'csi.quobyte.com'}
 # do not tag version as latest - that would trigger pull always from registry
 CSI_DRIVER_VERSION="$(git rev-parse --short HEAD)"
@@ -46,7 +46,7 @@ echo ""
 tee -a "${TEST_CLUSTER_DIR}"/Dockerfile <<END
 FROM kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
 RUN apt-get -y update 
-RUN apt-get -y install wget && apt-get install -y git && apt install -y nano
+RUN apt-get -y install wget && apt-get install -y git && apt install -y nano dnsutils
 END
 
 echo "Building new image using the above Dockerfile. On the local machine: * * *"
@@ -55,49 +55,13 @@ docker build -t 'quobyte/kind-node-testing:v0' -f "${TEST_CLUSTER_DIR}"/Dockerfi
 
 docker image ls | grep 'quobyte/kind-node-testing:v0'
 
-tee -a "${TEST_CLUSTER_DIR}"/audit-policy.yaml <<END
-apiVersion: audit.k8s.io/v1
-kind: Policy
-rules:
-  - level: RequestResponse
-    verbs: ["watch"]
-    resources:
-      - group: "" # core
-        resources: ["pods"]
-END
-
 echo "Use the newly created image in the kind cluster configuration: * * * * * * * * *"
 tee -a "${TEST_CLUSTER_DIR}"/kind-config-testing.yaml <<END
 kind: Cluster
 name: ${KIND_CLUSTER_NAME}
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
-  kubeadmConfigPatches:
-  - |
-    kind: ClusterConfiguration
-    apiServer:
-        # enable auditing flags on the API server
-        extraArgs:
-          audit-log-path: /var/log/kubernetes/kube-apiserver-audit.log
-          audit-policy-file: /etc/kubernetes/policies/audit-policy.yaml
-        # mount new files / directories on the control plane
-        extraVolumes:
-          - name: audit-policies
-            hostPath: /etc/kubernetes/policies
-            mountPath: /etc/kubernetes/policies
-            readOnly: true
-            pathType: "DirectoryOrCreate"
-          - name: "audit-logs"
-            hostPath: "/var/log/kubernetes"
-            mountPath: "/var/log/kubernetes"
-            readOnly: false
-            pathType: DirectoryOrCreate
-  # mount the local file on the control plane
-  extraMounts:
-  - hostPath: "./${TEST_CLUSTER_DIR}/audit-policy.yaml"
-    containerPath: /etc/kubernetes/policies/audit-policy.yaml
-    readOnly: true
 - role: worker
   image: quobyte/kind-node-testing:v0
 - role: worker
@@ -109,7 +73,50 @@ END
 echo  "Creating cluster with kind: * * * * * * * * * * * * * * * * * * * * * * *"
 kind create cluster --config="${TEST_CLUSTER_DIR}"/kind-config-testing.yaml --kubeconfig ${KUBECONFIG}
 
-echo  "Labeling Kubernetes nodes * * * * * * * * * * * * * * * * * * * * * * * *"
+dns_ips="$(kubectl get services -n kube-system | grep 'kube-dns' | awk '{print $3}')"
+if [[ -z "$dns_ips" ]]; then
+  echo "Error: could not retrive kube-dns IP required"
+  exit 1
+fi
+if ! [[ "$dns_ips" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  echo "Error: Invalid kube-dns IP $dns_ips"
+  exit 1
+fi
+
+# Add host dns entries, so that we can resolve all the names (dependent container urls, Quobyte
+# api/registry etc)
+for host_dns_ip in $(grep nameserver /etc/resolv.conf | awk '{print $2}'); do
+   dns_ips="$dns_ips $host_dns_ip"
+done
+
+# Requires for Quobyte pod killer components - cache is run as a service and mount monitor
+# uses service url to communicate with the cache. Without kube-dns mapped, mount monitor cannot
+# resolve the service url
+echo "Mapping k8s kube-dns resolver into k8s nodes * * * * * * * * *"
+for node in $(kubectl get nodes --no-headers | grep 'worker' | awk '{print $1}'); do
+  echo "Mapping k8s kube-dns resolver into $node"
+  # Nullify current configuration -- ATM, kind configures dns is configured with the default
+  # gateway of the virtual network
+  sudo docker exec -it $node sh -c "cat /dev/null > /etc/resolv.conf"
+  # Next couple of lines are not intended on purpose - otherwise command does not work as
+  # -c interprets text verbatim
+  sudo docker exec -it $node sh -c "cat > /etc/resolv.conf <<END
+search default svc.cluster.local cluster.local
+END"
+for dns_ip in $dns_ips; do
+sudo docker exec -it $node sh -c "cat >> /etc/resolv.conf <<END
+nameserver $dns_ip
+END"
+done
+ sudo docker exec -it $node sh -c "cat >> /etc/resolv.conf <<END
+options timeout:1 ndots:0
+END"
+  echo "Configured worker node $node with DNS entries as following:"
+  sudo docker exec -it quobyte-csi-testing-worker sh -c "cat /etc/resolv.conf"
+  echo ""
+done
+
+echo  "Labeling Kubernetes nodes for Quobyte client installation * * * * * * * * * * * * * * *"
 for node in $(kubectl get nodes --no-headers | awk '{print $1}'); do
   kubectl label nodes $node quobyte_client="true"
 done