MultiZones support

Add support for multiple zones depresolver move config to gslb instead of edgeDNSServers getNSNames fix tests selecting GSLB by stateus.Servers[*].Host fixing terratest Fix externalDNSENdpoint name , IBX finalizer fix infoblox finalizer test replace config.DNSZone by config.DelegationZones[*].Domain use config.DelegationZones[*].Zone instead of EdgeDNSZone Signed-off-by: Michal Kuritka <[email protected]> making egdeDNSZone private, so it cant be used cross the project but is still there for backward compatibility making dnsZone private, so it cant be used cross the project but is still there for backward compatibility Move GetClusterNSNames to DelegationZones GetExternalClusterNSNamesByHostname making getClusterNSName and getExternalClusterNSNames private so it can't be used anymore extClustersGeoTags field is private update helm k8gb extend helm for coredns cm
k8gb-io · Feb 20, 2025 · 2325220 · 2325220
1 parent cb64ca4
commit 2325220
Show file tree

Hide file tree

Showing 25 changed files with 832 additions and 267 deletions.
diff --git a/chart/k8gb/templates/_helpers.tpl b/chart/k8gb/templates/_helpers.tpl
@@ -96,6 +96,28 @@ k8gb-{{ .Values.k8gb.dnsZone }}-{{ .Values.k8gb.clusterGeoTag }}
 {{- end -}}
 {{- end -}}
 
+{{- define "k8gb.dnsZones" -}}
+{{- $entries := list -}}
+{{- range .Values.k8gb.dnsZones }}
+  {{- $entry := printf "%s:%s" .zone .domain }}
+  {{- $entries = append $entries $entry }}
+{{- end }}
+{{- if and .Values.k8gb.dnsZone .Values.k8gb.edgeDNSZone }}
+  {{- $extraEntry := printf "%s:%s" .Values.k8gb.edgeDNSZone .Values.k8gb.dnsZone }}
+  {{- $entries = append $entries $extraEntry }}
+{{- end }}
+{{- join ";" $entries }}
+{{- end }}
+
+
+{{- define "k8gb.coredns.extraPlugins" -}}
+{{- if .Values.k8gb.coredns.extra_plugins }}
+{{- range .Values.k8gb.coredns.extra_plugins }}
+        {{ . }}
+{{- end }}
+{{- end }}
+{{- end }}
+
 {{- define "k8gb.extdnsProviderOpts" -}}
 {{- if .Values.ns1.enabled -}}
 {{- if .Values.ns1.endpoint -}}

diff --git a/chart/k8gb/templates/coredns/cm.yaml b/chart/k8gb/templates/coredns/cm.yaml
@@ -8,11 +8,12 @@ metadata:
 {{ include "chart.labels" . | indent 4  }}
 data:
   Corefile: |-
-    {{ .Values.k8gb.dnsZone }}:5353 {
+{{- range .Values.k8gb.dnsZones }}
+    {{ .domain }}:5353 {
         errors
         health
-{{- if .Values.k8gb.coredns.extra_plugins }}
-{{- range .Values.k8gb.coredns.extra_plugins }}
+{{- if $.Values.k8gb.coredns.extra_plugins }}
+{{- range $.Values.k8gb.coredns.extra_plugins }}
         {{ . }}
 {{- end }}
 {{- end }}
@@ -21,10 +22,11 @@ data:
         forward . /etc/resolv.conf
         k8s_crd {
             filter k8gb.absa.oss/dnstype=local
-            negttl {{ .Values.k8gb.dnsZoneNegTTL }}
+            negttl {{ .dnsZoneNegTTL | default 30 }}
             loadbalance weight
         }
     }
+{{- end }}
     {{- with .Values.k8gb.coredns.extraServerBlocks -}}
     {{- tpl . $ | nindent 4 }}
     {{- end }}

diff --git a/chart/k8gb/templates/deployment.yaml b/chart/k8gb/templates/deployment.yaml
@@ -59,12 +59,10 @@ spec:
               value: {{ quote .Values.k8gb.clusterGeoTag }}
             - name: EXT_GSLB_CLUSTERS_GEO_TAGS
               value: {{ quote .Values.k8gb.extGslbClustersGeoTags }}
-            - name: EDGE_DNS_ZONE
-              value: {{ .Values.k8gb.edgeDNSZone }}
             - name: EDGE_DNS_SERVERS
               value: {{ include "k8gb.edgeDNSServers" . }}
-            - name: DNS_ZONE
-              value: {{ .Values.k8gb.dnsZone }}
+            - name: DNS_ZONES
+              value: {{ include "k8gb.dnsZones" . }}
             - name: RECONCILE_REQUEUE_SECONDS
               value: {{ quote .Values.k8gb.reconcileRequeueSeconds }}
             - name: NS_RECORD_TTL

diff --git a/chart/k8gb/values.schema.json b/chart/k8gb/values.schema.json
@@ -266,6 +266,12 @@
                 "deployRbac": {
                     "type": "boolean"
                 },
+                "dnsZones": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/k8gbDnsZone"
+                    }
+                },
                 "dnsZone": {
                     "format": "idn-hostname",
                     "minLength": 1
@@ -334,12 +340,24 @@
                     "type": "object"
                 }
             },
-            "required": [
-                "clusterGeoTag",
-                "extGslbClustersGeoTags",
-                "dnsZone",
-                "edgeDNSServers",
-                "edgeDNSZone"
+            "oneOf": [
+                {
+                    "required": [
+                        "clusterGeoTag",
+                        "extGslbClustersGeoTags",
+                        "edgeDNSServers",
+                        "dnsZone",
+                        "edgeDNSZone"
+                    ]
+                },
+                {
+                    "required": [
+                        "clusterGeoTag",
+                        "extGslbClustersGeoTags",
+                        "edgeDNSServers",
+                        "dnsZones"
+                    ]
+                }
             ],
             "title": "k8gb"
         },
@@ -421,6 +439,28 @@
                 }
             }
         },
+        "k8gbDnsZone": {
+            "type": "object",
+            "properties": {
+                "zone": {
+                    "type": "string",
+                    "format": "idn-hostname"
+                },
+                "domain": {
+                    "type": "string",
+                    "format": "idn-hostname"
+                },
+                "dnsZoneNegTTL": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 300
+                }
+            },
+            "required": [
+                "zone",
+                "domain"
+            ]
+        },
         "Ns1": {
             "type": "object",
             "additionalProperties": false,

diff --git a/chart/k8gb/values.yaml b/chart/k8gb/values.yaml
@@ -13,12 +13,17 @@ k8gb:
   deployCrds: true
   # -- whether it should also deploy the service account, cluster role and cluster role binding
   deployRbac: true
-  # -- dnsZone controlled by gslb
-  dnsZone: "cloud.example.com"
-  # -- Negative TTL for SOA record
-  dnsZoneNegTTL: 300
-  # -- main zone which would contain gslb zone to delegate
-  edgeDNSZone: "example.com" # main zone which would contain gslb zone to delegate
+  # -- deprecated: dnsZone controlled by gslb
+  # dnsZone: "cloud.example.org"
+  # -- deprecated: main zone which would contain gslb zone to delegate
+  # edgeDNSZone: example.org # main zone which would contain gslb zone to delegate
+  # -- array of dns zones controlled by gslb
+  dnsZones:
+    - zone: "example.com" # -- main zone which would contain gslb zone to delegate (same meaning as to edgeDNSZone)
+      domain: "cloud.example.com" # -- domain controlled by gslb (same meaning as to dnsZone)
+      dnsZoneNegTTL: 300  # -- Negative TTL for SOA record
+    - zone: "example.org" # -- main zone which would contain gslb zone to delegate (same meaning as to edgeDNSZone)
+      domain: "cloud.example.org" # -- domain controlled by gslb (same meaning as to dnsZone)
   # -- host/ip[:port] format is supported here where port defaults to 53
   edgeDNSServers:
     # -- use this DNS server as a main resolver to enable cross k8gb DNS based communication

diff --git a/controllers/depresolver/depresolver.go b/controllers/depresolver/depresolver.go
@@ -124,8 +124,8 @@ type Config struct {
 	NSRecordTTL int `env:"NS_RECORD_TTL, default=30"`
 	// ClusterGeoTag to determine specific location
 	ClusterGeoTag string `env:"CLUSTER_GEO_TAG"`
-	// ExtClustersGeoTags to identify clusters in other locations in format separated by comma. i.e.: "eu,uk,us"
-	ExtClustersGeoTags []string `env:"EXT_GSLB_CLUSTERS_GEO_TAGS, default=[]"`
+	// extClustersGeoTags to identify clusters in other locations in format separated by comma. i.e.: "eu,uk,us"
+	extClustersGeoTags []string `env:"EXT_GSLB_CLUSTERS_GEO_TAGS, default=[]"`
 	// EdgeDNSType is READONLY and is set automatically by configuration
 	EdgeDNSType EdgeDNSType
 	// EdgeDNSServers
@@ -134,10 +134,14 @@ type Config struct {
 	fallbackEdgeDNSServerName string `env:"EDGE_DNS_SERVER"`
 	// to avoid breaking changes is used as fallback server port for EdgeDNSServers
 	fallbackEdgeDNSServerPort int `env:"EDGE_DNS_SERVER_PORT, default=53"`
-	// EdgeDNSZone main zone which would contain gslb zone to delegate; e.g. example.com
-	EdgeDNSZone string `env:"EDGE_DNS_ZONE"`
-	// DNSZone controlled by gslb; e.g. cloud.example.com
-	DNSZone string `env:"DNS_ZONE"`
+	// edgeDNSZone main zone which would contain gslb zone to delegate; e.g. example.com
+	edgeDNSZone string `env:"EDGE_DNS_ZONE"`
+	// dnsZone controlled by gslb; e.g. cloud.example.com
+	dnsZone string `env:"DNS_ZONE"`
+	// DelegationZones
+	DelegationZones DelegationZones
+	// DelegationZones pairs of dnsZone ad edgeDNSZone, eg: DNS_ZONES=example.com:cloud.example.com;example.io:cloud.example.io
+	dnsZones string `env:"DNS_ZONES"`
 	// K8gbNamespace k8gb namespace
 	K8gbNamespace string `env:"POD_NAMESPACE"`
 	// Infoblox configuration

diff --git a/controllers/depresolver/depresolver_config.go b/controllers/depresolver/depresolver_config.go
@@ -31,6 +31,9 @@ import (
 	"github.com/rs/zerolog"
 )
 
+// TODO: refactor with kong.CLI to read envvars into Config
+// TODO: refactor with go-playground/validator to validate
+
 // Environment variables keys
 const (
 	ReconcileRequeueSecondsKey = "RECONCILE_REQUEUE_SECONDS"
@@ -41,6 +44,7 @@ const (
 	EdgeDNSServersKey          = "EDGE_DNS_SERVERS"
 	EdgeDNSZoneKey             = "EDGE_DNS_ZONE"
 	DNSZoneKey                 = "DNS_ZONE"
+	DNSZonesKey                = "DNS_ZONES"
 	InfobloxGridHostKey        = "INFOBLOX_GRID_HOST"
 	InfobloxVersionKey         = "INFOBLOX_WAPI_VERSION"
 	InfobloxPortKey            = "INFOBLOX_WAPI_PORT"
@@ -69,6 +73,12 @@ const (
 	EdgeDNSServerPortKey = "EDGE_DNS_SERVER_PORT"
 )
 
+const (
+	localhost = "localhost"
+
+	localhostIPv4 = "127.0.0.1"
+)
+
 // ResolveOperatorConfig executes once. It reads operator's configuration
 // from environment variables into &Config and validates
 func (dr *DependencyResolver) ResolveOperatorConfig() (*Config, error) {
@@ -87,13 +97,17 @@ func (dr *DependencyResolver) ResolveOperatorConfig() (*Config, error) {
 		fallbackDNS := fmt.Sprintf("%s:%v", dr.config.fallbackEdgeDNSServerName, dr.config.fallbackEdgeDNSServerPort)
 		edgeDNSServerList := env.GetEnvAsArrayOfStringsOrFallback(EdgeDNSServersKey, []string{fallbackDNS})
 		dr.config.EdgeDNSServers = parseEdgeDNSServers(edgeDNSServerList)
-		dr.config.ExtClustersGeoTags = excludeGeoTag(dr.config.ExtClustersGeoTags, dr.config.ClusterGeoTag)
+		dr.config.extClustersGeoTags = excludeGeoTag(dr.config.extClustersGeoTags, dr.config.ClusterGeoTag)
 		dr.config.Log.Level, _ = zerolog.ParseLevel(strings.ToLower(dr.config.Log.level))
 		dr.config.Log.Format = parseLogOutputFormat(strings.ToLower(dr.config.Log.format))
 		dr.config.EdgeDNSType, recognizedDNSTypes = getEdgeDNSType(dr.config)
 
-		// validation
+		// replace validations by go-playground/validator
 		dr.errorConfig = dr.validateConfig(dr.config, recognizedDNSTypes)
+		// validation
+		if dr.errorConfig == nil {
+			dr.config.DelegationZones = parseDelegationZones(dr.config)
+		}
 	})
 	return dr.config, dr.errorConfig
 }
@@ -128,11 +142,11 @@ func (dr *DependencyResolver) validateConfig(config *Config, recognizedDNSTypes
 	if err != nil {
 		return err
 	}
-	err = field(ExtClustersGeoTagsKey, config.ExtClustersGeoTags).hasItems().hasUniqueItems().err
+	err = field(ExtClustersGeoTagsKey, config.extClustersGeoTags).hasItems().hasUniqueItems().err
 	if err != nil {
 		return err
 	}
-	for i, geoTag := range config.ExtClustersGeoTags {
+	for i, geoTag := range config.extClustersGeoTags {
 		err = field(fmt.Sprintf("%s[%v]", ExtClustersGeoTagsKey, i), geoTag).
 			isNotEmpty().matchRegexp(geoTagRegex).err
 		if err != nil {
@@ -160,11 +174,11 @@ func (dr *DependencyResolver) validateConfig(config *Config, recognizedDNSTypes
 			return fmt.Errorf("error for port of edge dns server(%v): it must be a positive integer between 1 and 65535", s)
 		}
 	}
-	err = field(EdgeDNSZoneKey, config.EdgeDNSZone).isNotEmpty().matchRegexp(hostNameRegex).err
+	err = field(EdgeDNSZoneKey, config.edgeDNSZone).isNotEmpty().matchRegexp(hostNameRegex).err
 	if err != nil {
 		return err
 	}
-	err = field(DNSZoneKey, config.DNSZone).isNotEmpty().matchRegexp(hostNameRegex).err
+	err = field(DNSZoneKey, config.dnsZone).isNotEmpty().matchRegexp(hostNameRegex).err
 	if err != nil {
 		return err
 	}
@@ -185,12 +199,12 @@ func (dr *DependencyResolver) validateConfig(config *Config, recognizedDNSTypes
 		return nil
 	}
 
-	serverNames := config.GetExternalClusterNSNames()
-	serverNames[config.ClusterGeoTag] = config.GetClusterNSName()
+	serverNames := config.getExternalClusterNSNames()
+	serverNames[config.ClusterGeoTag] = config.getClusterNSName()
 	for geoTag, nsName := range serverNames {
 		if len(nsName) > dnsNameMax {
 			return fmt.Errorf("ns name '%s' exceeds %v charactes limit for [GeoTag: '%s', %s: '%s', %s: '%s']",
-				nsName, dnsLabelMax, geoTag, EdgeDNSZoneKey, config.EdgeDNSZone, DNSZoneKey, config.DNSZone)
+				nsName, dnsLabelMax, geoTag, EdgeDNSZoneKey, config.edgeDNSZone, DNSZoneKey, config.dnsZone)
 		}
 		if err := validateLabels(nsName); err != nil {
 			return fmt.Errorf("error for geo tag: %s. %s in ns name %s", geoTag, err, nsName)
@@ -215,7 +229,7 @@ func (dr *DependencyResolver) validateConfig(config *Config, recognizedDNSTypes
 func validateLocalhostNotAmongDNSServers(config *Config) error {
 	containsLocalhost := func(list utils.DNSList) bool {
 		for i := 1; i < len(list); i++ { // skipping first because localhost or 127.0.0.1 can occur on the first position
-			if list[i].Host == "localhost" || list[i].Host == "127.0.0.1" {
+			if list[i].Host == localhost || list[i].Host == localhostIPv4 {
 				return true
 			}
 		}
@@ -370,20 +384,22 @@ func parseLogOutputFormat(value string) LogFormat {
 	return NoFormat
 }
 
-func (c *Config) GetExternalClusterNSNames() (m map[string]string) {
-	m = make(map[string]string, len(c.ExtClustersGeoTags))
-	for _, tag := range c.ExtClustersGeoTags {
-		m[tag] = getNsName(tag, c.DNSZone, c.EdgeDNSZone, c.EdgeDNSServers[0].Host)
+// deprecated, used for validations only
+func (c *Config) getExternalClusterNSNames() (m map[string]string) {
+	m = make(map[string]string, len(c.extClustersGeoTags))
+	for _, tag := range c.extClustersGeoTags {
+		m[tag] = getNsName(tag, c.dnsZone, c.edgeDNSZone, c.EdgeDNSServers[0].Host)
 	}
 	return
 }
 
-func (c *Config) GetClusterNSName() string {
-	return getNsName(c.ClusterGeoTag, c.DNSZone, c.EdgeDNSZone, c.EdgeDNSServers[0].Host)
+// deprecated, used for validations only
+func (c *Config) getClusterNSName() string {
+	return getNsName(c.ClusterGeoTag, c.dnsZone, c.edgeDNSZone, c.EdgeDNSServers[0].Host)
 }
 
 // getNsName returns NS for geo tag.
-// The values is combination of DNSZone, EdgeDNSZone and (Ext)ClusterGeoTag, see:
+// The values is combination of dnsZone, edgeDNSZone and (Ext)ClusterGeoTag, see:
 // DNS_ZONE k8gb-test.gslb.cloud.example.com
 // EDGE_DNS_ZONE: cloud.example.com
 // CLUSTER_GEOTAG: us