expose coredns as loadbalancer service in local setup

Signed-off-by: Andre Aguas <[email protected]>

Makefile

@@ -159,7 +159,6 @@ deploy-test-version: ## Upgrade k8gb to the test version on existing clusters
 
 	@for c in $(CLUSTER_IDS); do \
 		$(MAKE) deploy-local-cluster CLUSTER_ID=$$c VERSION=$(SEMVER)-$(ARCH) CHART='./chart/k8gb' ;\
-		kubectl apply -n k8gb -f ./deploy/test/coredns-tcp-svc.yaml ;\
 	done
 
 .PHONY: list-running-pods
@@ -253,8 +252,10 @@ deploy-k8gb-with-helm:
 	kubectl -n k8gb create secret generic rfc2136 --from-literal=secret=96Ah/a2g0/nLeFGK+d/0tzQcccf9hCEIy34PoXX2Qg8= || true
 	helm repo add --force-update k8gb https://www.k8gb.io
 	cd chart/k8gb && helm dependency update
+	kubectl -n k8gb delete svc k8gb-coredns --ignore-not-found
 	helm -n k8gb upgrade -i k8gb $(CHART) -f $(VALUES_YAML) \
 		--set $(call get-helm-args,$(CLUSTER_ID)) \
+		--set coredns.serviceType=LoadBalancer \
 		--set k8gb.reconcileRequeueSeconds=10 \
 		--set k8gb.dnsZoneNegTTL=10 \
 		--set k8gb.imageTag=${VERSION:"stable"=""} \

controllers/providers/assistant/gslb.go

@@ -118,8 +118,16 @@ func (r *Gslb) CoreDNSExposedIPs() ([]string, error) {
 		err := coreerrors.New(errMessage)
 		return nil, err
 	}
-	lb = coreDNSService.Status.LoadBalancer.Ingress[0]
-	return extractIPFromLB(lb, r.edgeDNSServers)
+	lbIPs := []string{}
+	for _, lb = range coreDNSService.Status.LoadBalancer.Ingress {
+		ips, err := extractIPFromLB(lb, r.edgeDNSServers)
+		lbIPs = append(lbIPs, ips...)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return lbIPs, nil
 }
 
 func extractIPFromLB(lb corev1.LoadBalancerIngress, ns utils.DNSList) (ips []string, err error) {

deploy/ingress/nginx-ingress-values.yaml

@@ -21,9 +21,3 @@ controller:
 
   service:
     enabled: false
-
-# UDP service key:value pairs
-# Ref: https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx/examples/udp
-##
-udp:
-  53: "k8gb/k8gb-coredns:53"

k3d/test-gslb1.yaml

@@ -23,15 +23,13 @@ ports:
   - port: 9080:30090
     nodeFilters:
       - agent:0:direct
-  - port: 5053:30053/tcp
+  - port: 5053:53/tcp
     nodeFilters:
-      - agent:0:direct
+      - loadbalancer
   - port: 5053:53/udp
     nodeFilters:
-      - agent:0:direct
+      - loadbalancer
 options:
-  k3d:
-    disableLoadbalancer: true
   k3s:
     extraArgs:
       - arg: --disable=traefik,metrics-server,local-storage

k3d/test-gslb2.yaml

@@ -20,15 +20,13 @@ ports:
   - port: 9081:30090
     nodeFilters:
       - agent:0:direct
-  - port: 5054:30053/tcp
+  - port: 5054:53/tcp
     nodeFilters:
-      - agent:0:direct
+      - loadbalancer
   - port: 5054:53/udp
     nodeFilters:
-      - agent:0:direct
+      - loadbalancer
 options:
-  k3d:
-    disableLoadbalancer: true
   k3s:
     extraArgs:
       - arg: --disable=traefik,metrics-server,local-storage

k3d/test-gslb3.yaml

@@ -20,15 +20,13 @@ ports:
   - port: 9082:30090
     nodeFilters:
       - agent:0:direct
-  - port: 5055:30053/tcp
+  - port: 5055:53/tcp
     nodeFilters:
-      - agent:0:direct
+      - loadbalancer
   - port: 5055:53/udp
     nodeFilters:
-      - agent:0:direct
+      - loadbalancer
 options:
-  k3d:
-    disableLoadbalancer: true
   k3s:
     extraArgs:
       - arg: --disable=traefik,metrics-server,local-storage

terratest/test/k8gb_weight_test.go

@@ -19,6 +19,7 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic
 */
 
 import (
+	"fmt"
 	"testing"
 
 	"k8gbterratest/utils"
@@ -69,6 +70,9 @@ func abstractTestWeightsExistsInLocalDNSEndpoint(t *testing.T, host string, work
 	epExternalUS, err := instanceUS.Resources().GetK8gbExternalDNSEndpoint().GetEndpointByName(endpointDNSNameUS)
 	require.NoError(t, err, "missing US endpoint %s", endpointDNSNameUS)
 
+	fmt.Println("EU targets", epExternalEU.Targets)
+	fmt.Println("US targets", epExternalUS.Targets)
+
 	expectedTargets := append(epExternalEU.Targets, epExternalUS.Targets...)
 
 	err = instanceEU.WaitForLocalDNSEndpointHasTargets(expectedTargets)

terratest/utils/extensions.go

@@ -479,6 +479,8 @@ func (i *Instance) WaitForLocalDNSEndpointHasTargets(expectedIPs []string) error
 	periodic := func() (result bool, err error) {
 		lep := i.Resources().GetLocalDNSEndpoint()
 		ep, err := lep.GetEndpointByName(i.w.state.gslb.host)
+		fmt.Println("LocalDNSEndpoint targets:", ep.Targets)
+		fmt.Println("Expected targets:", expectedIPs)
 		result = EqualStringSlices(ep.Targets, expectedIPs)
 		return result, err
 	}