Switch to coredns with DNSendpoint plugin

Makefile

@@ -10,7 +10,6 @@ VALUES_YAML ?= chart/k8gb/values.yaml
 PODINFO_IMAGE_REPO ?= stefanprodan/podinfo
 HELM_ARGS ?=
 K8GB_COREDNS_IP ?= kubectl get svc k8gb-coredns -n k8gb -o custom-columns='IP:spec.clusterIP' --no-headers
-ETCD_DEBUG_IMAGE ?= quay.io/coreos/etcd:v3.2.25
 
 CLUSTER_GSLB2_HELM_ARGS ?= --set k8gb.clusterGeoTag='us' --set k8gb.extGslbClustersGeoTags='eu' --set k8gb.hostAlias.hostnames='{gslb-ns-cloud-example-com-eu.example.com}'
 GITACTION_IMAGE_REPO ?=registry.localhost:5000/k8gb
@@ -77,10 +76,6 @@ debug-idea: export WATCH_NAMESPACE=test-gslb
 debug-idea:
 	$(call debug,debug --headless --listen=:2345 --api-version=2)
 
-.PHONY: debug-test-etcd
-debug-test-etcd: ## Run temporary etcd pod for debug
-	kubectl run --rm -i --tty --env="ETCDCTL_API=3" --env="ETCDCTL_ENDPOINTS=http://etcd-cluster-client:2379" --namespace k8gb etcd-test --image "$(ETCD_DEBUG_IMAGE)" --restart=Never -- /bin/sh
-
 .PHONY: demo-roundrobin
 demo-roundrobin: ## Execute round-robin demo
 	@$(call demo-host, "roundrobin.cloud.example.com")
@@ -339,7 +334,7 @@ define deploy-local-cluster
 	@echo "\n$(YELLOW)Deploy test apps $(NC)"
 	$(call deploy-test-apps)
 
-	@echo "\n$(YELLOW)Wait until ETCD and Ingress controller are ready $(NC)"
+	@echo "\n$(YELLOW)Wait until Ingress controller is ready $(NC)"
 	$(call wait)
 
 	@echo "\n$(CYAN)$1 $(YELLOW)deployed! $(NC)"
@@ -404,7 +399,6 @@ endef
 # waits for NGINX, GSLB are ready
 define wait
 	kubectl -n k8gb wait --for=condition=Ready pod -l app=nginx-ingress --timeout=600s
-	kubectl -n k8gb wait --for=condition=Ready pod -l app=etcd --timeout=600s
 endef
 
 define generate

chart/k8gb/Chart.lock

@@ -2,8 +2,5 @@ dependencies:
 - name: coredns
   repository: https://coredns.github.io/helm
   version: 1.14.0
-- name: etcd-operator
-  repository: https://charts.helm.sh/stable
-  version: 0.11.0
-digest: sha256:a424dc6f1d74fd5aa194629fcccb62a1ee02e9a5317ed20756245e66e80dfe5a
-generated: "2020-12-16T20:55:58.809683+01:00"
+digest: sha256:3a6b4ca5f4e2be5a9d5c65320b975001f14a02a7eb258f922e68a9f37013e4db
+generated: "2021-02-05T20:59:23.594399+01:00"

chart/k8gb/Chart.yaml

@@ -9,9 +9,6 @@ dependencies:
   - name: coredns
     repository: https://coredns.github.io/helm
     version: 1.14.0
-  - name: etcd-operator
-    repository: https://charts.helm.sh/stable
-    version: 0.11.0
 
 home: https://www.k8gb.io/
 sources:
@@ -51,4 +48,4 @@ annotations:
                   path: /
         strategy:
           type: failover # Global load balancing strategy
-          primaryGeoTag: eu-west-1 # Primary cluster geo tag
+          primaryGeoTag: eu-west-1 # Primary cluster geo tag

chart/k8gb/templates/NOTES.txt

@@ -1,12 +1,6 @@
 K8GB and all deps are installed
 
-1. Watch etcd cluster start
-  $ kubectl get pods -l etcd_cluster=etcd-cluster --namespace {{ .Release.Namespace }} -w
-
-2. Confirm etcd cluster is healthy
-  $ kubectl run --rm -i --tty --env="ETCDCTL_API=3" --env="ETCDCTL_ENDPOINTS=http://etcd-cluster-client:2379" --namespace {{ .Release.Namespace }} etcd-test --image quay.io/coreos/etcd --restart=Never -- /bin/sh -c 'watch -n1 "etcdctl  member list"'
-
-3. Check if your DNS Zone is served by K8GB CoreDNS
+1. Check if your DNS Zone is served by K8GB CoreDNS
   $ kubectl -n {{ .Release.Namespace }} run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools --command -- /usr/bin/dig @{{ .Release.Name }}-coredns SOA {{ .Values.k8gb.dnsZone }} +short
 
 If everything is fine than you are expected to see similar output:

chart/k8gb/templates/coredns/rbac.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: coredns-cluster-role 
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - externaldns.k8s.io
+  resources:
+  - dnsendpoints 
+  verbs:
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: coredns-clusterrole-binding 
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: coredns-cluster-role 
+subjects:
+- kind: ServiceAccount
+  name: coredns 
+  namespace: {{ .Release.Namespace }}

chart/k8gb/values.yaml

@@ -26,33 +26,14 @@ externaldns:
   interval: "20s"
   expose53onWorkers: true # open 53/udp on workers nodes with nginx controller
 
-etcd-operator:
-  customResources:
-    createEtcdClusterCRD: true
-  etcdOperator:
-    image:
-      repository: quay.io/coreos/etcd-operator
-      tag: v0.9.4
-  backupOperator:
-    image:
-      repository: quay.io/coreos/etcd-operator
-      tag: v0.9.4
-  restoreOperator:
-    image:
-      repository: quay.io/coreos/etcd-operator
-      tag: v0.9.4
-  etcdCluster:
-    image:
-      repository: quay.io/coreos/etcd
-    version: 3.2.25
-    pod:
-      busyboxImage: busybox:1.28.0-glibc
-
 coredns:
   isClusterService: false
   image:
-    repository: coredns/coredns
-    tag: "1.6.6"
+    repository: absaoss/k8s_crd
+    tag: "v0.0.1"
+  serviceAccount:
+    create: true
+    name: coredns
   servers:
   - zones:
     - zone: .
@@ -65,22 +46,18 @@ coredns:
     - name: health
     # Serves a /ready endpoint on :8181, required for readinessProbe
     - name: ready
-    # Required to query kubernetes API for data
-    - name: kubernetes
-      parameters: cluster.local
     - name: loadbalance
       parameters: round_robin
     # Serves a /metrics endpoint on :9153, required for serviceMonitor
     - name: prometheus
       parameters: 0.0.0.0:9153
     - name: forward
       parameters: . /etc/resolv.conf
-    - name: etcd
+    - name: k8s_crd
       parameters: .
       configBlock: |-
-        stubzones
-        path /skydns
-        endpoint http://etcd-cluster-client:2379
+        resources DNSEndpoint
+        filter k8gb.absa.oss/dnstype=local
 
 infoblox:
   enabled: false

controllers/dnsupdate.go

@@ -94,6 +94,7 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D
 			Name:        gslb.Name,
 			Namespace:   gslb.Namespace,
 			Annotations: map[string]string{"k8gb.absa.oss/dnstype": "local"},
+			Labels:      map[string]string{"k8gb.absa.oss/dnstype": "local"},
 		},
 		Spec: dnsEndpointSpec,
 	}

docs/deploy_infoblox.md

@@ -45,29 +45,9 @@ make deploy-gslb-operator VALUES_YAML=~/k8gb/eu-cluster.yaml
 ```sh
  kubectl -n k8gb get pod
 NAME                                                       READY   STATUS     RESTARTS   AGE
-etcd-cluster-gnt7rslvvj                                    1/1     Running    0          20s
-etcd-cluster-n8b2rvhv5q                                    0/1     Init:0/1   0          3s
 external-dns-79d5ccd7fc-4bj74                              1/1     Running    0          39s
 k8gb-76cc56b55-t779s                                       1/1     Running    0          39s
 k8gb-coredns-799984c646-qz88m                              1/1     Running    0          41s
-k8gb-etcd-operator-etcd-backup-operator-fccb8fdd-bx8v5     1/1     Running    0          40s
-k8gb-etcd-operator-etcd-operator-765f655575-pgvbh          1/1     Running    0          40s
-k8gb-etcd-operator-etcd-restore-operator-8977cd5c8-k8l7l   1/1     Running    0          40s
-```
-
-* As you can see one of the `etcd-cluster` pods is still in `Init` state. The embedded `etcd-operator` is automatically configuring isolated `etcd` cluster to act as a backend for `coredns`, which is used as part of k8gb for coordinated DNS responses. Expected post installation result:
-```sh
-kubectl -n k8gb get pod
-NAME                                                       READY   STATUS    RESTARTS   AGE
-etcd-cluster-gnt7rslvvj                                    1/1     Running   0          7m39s
-etcd-cluster-n8b2rvhv5q                                    1/1     Running   0          7m22s
-etcd-cluster-xmkz8xsk7g                                    1/1     Running   0          6m41s
-external-dns-79d5ccd7fc-4bj74                              1/1     Running   0          7m58s
-k8gb-76cc56b55-t779s                                       1/1     Running   0          7m58s
-k8gb-coredns-799984c646-qz88m                              1/1     Running   0          8m
-k8gb-etcd-operator-etcd-backup-operator-fccb8fdd-bx8v5     1/1     Running   0          7m59s
-k8gb-etcd-operator-etcd-operator-765f655575-pgvbh          1/1     Running   0          7m59s
-k8gb-etcd-operator-etcd-restore-operator-8977cd5c8-k8l7l   1/1     Running   0          7m59s
 ```
 
 * Deploy k8gb to the second cluster by repeating the same steps with the exception of: