feat!: Separate out internal and external nginx controllers

README.md

@@ -359,7 +359,7 @@ _Mic check, 1, 2_ - In a few moments applications should be lighting up like Chr
 
 #### 🌐 DNS
 
-The `external-dns` application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only public sub-domains exposed. In order to make additional applications public you must set an ingress annotation (`external-dns.alpha.kubernetes.io/target`) like done in the `HelmRelease` for `echo-server`.
+The `external-dns` application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only public sub-domains exposed. In order to make additional applications public you must set the correct ingress class name and ingress annotations like done in the `HelmRelease` for `echo-server`.
 
 For split DNS to work it is required to have `${bootstrap_cloudflare_domain}` point to the `${bootstrap_k8s_gateway_addr}` load balancer IP address on your home DNS server. This will ensure DNS requests for `${bootstrap_cloudflare_domain}` will only get routed to your `k8s_gateway` service thus providing **internal** DNS resolution to your cluster applications/ingresses from any device that uses your home DNS server.
 
@@ -370,7 +370,7 @@ For and example with Pi-Hole apply the following file and restart dnsmasq:
 server=/${bootstrap_cloudflare_domain}/${bootstrap_k8s_gateway_addr}
 ```
 
-Now try to resolve an internal-only domain with `dig @${pi-hole-ip} hajimari.${bootstrap_cloudflare_domain}` it should resolve to your `${bootstrap_ingress_nginx_addr}` IP.
+Now try to resolve an internal-only domain with `dig @${pi-hole-ip} hajimari.${bootstrap_cloudflare_domain}` it should resolve to your `${bootstrap_internal_nginx_addr}` IP.
 
 If you're having trouble with DNS be sure to check out these two Github discussions, [Internal DNS](https://github.com/onedr0p/flux-cluster-template/discussions/719) and [Pod DNS resolution broken](https://github.com/onedr0p/flux-cluster-template/discussions/635).
 

bootstrap/tasks/validation/net.yaml

@@ -67,17 +67,29 @@
     success_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is within {{ bootstrap_node_cidr }}.
     fail_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is not within {{ bootstrap_node_cidr }}.
 
-- name: Verify ingress-nginx
+- name: Verify internal nginx
   ansible.builtin.assert:
-    that: bootstrap_ingress_nginx_addr is ansible.utils.ipv4
-    success_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is valid.
-    fail_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is invalid.
+    that: bootstrap_internal_nginx_addr is ansible.utils.ipv4
+    success_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is valid.
+    fail_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is invalid.
 
-- name: Verify ingress-nginx in node CIDR
+- name: Verify internal nginx in node CIDR
   ansible.builtin.assert:
-    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_ingress_nginx_addr)
-    success_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is within {{ bootstrap_node_cidr }}.
-    fail_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is not within {{ bootstrap_node_cidr }}.
+    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_internal_nginx_addr)
+    success_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is within {{ bootstrap_node_cidr }}.
+    fail_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is not within {{ bootstrap_node_cidr }}.
+
+- name: Verify external nginx
+  ansible.builtin.assert:
+    that: bootstrap_external_nginx_addr is ansible.utils.ipv4
+    success_msg: external nginx address {{ bootstrap_external_nginx_addr }} is valid.
+    fail_msg: external nginx address {{ bootstrap_external_nginx_addr }} is invalid.
+
+- name: Verify external nginx in node CIDR
+  ansible.builtin.assert:
+    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_external_nginx_addr)
+    success_msg: external nginx address {{ bootstrap_external_nginx_addr }} is within {{ bootstrap_node_cidr }}.
+    fail_msg: external nginx address {{ bootstrap_external_nginx_addr }} is not within {{ bootstrap_node_cidr }}.
 
 - name: Verify kube-vip
   ansible.builtin.assert:
@@ -91,9 +103,21 @@
     success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}.
     fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}.
 
-- name: Verify nodes are not the same IPs as k8s_gateway, ingress-nginx or kube-vip
+- name: Verify all IP addresses are unique
+  ansible.builtin.assert:
+    that: >
+      [
+        bootstrap_k8s_gateway_addr,
+        bootstrap_external_nginx_addr,
+        bootstrap_internal_nginx_addr,
+        bootstrap_kube_vip_addr
+      ] | unique | length == 4
+    success_msg: All IP addresses are unique.
+    fail_msg: All IP addresses are not unique.
+
+- name: Verify nodes are not the same IPs as k8s_gateway, nginx external/internal or kube-vip
   ansible.builtin.assert:
-    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_ingress_nginx_addr, bootstrap_kube_vip_addr)
+    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_nginx_addr, bootstrap_internal_nginx_addr, bootstrap_kube_vip_addr)
     success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip.
     fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip.
     quiet: true

bootstrap/tasks/validation/vars.yaml

@@ -17,7 +17,8 @@
     - bootstrap_flux_github_webhook_token
     - bootstrap_github_repository_name
     - bootstrap_github_username
-    - bootstrap_ingress_nginx_addr
+    - bootstrap_external_nginx_addr
+    - bootstrap_internal_nginx_addr
     - bootstrap_ipv6_enabled
     - bootstrap_k8s_gateway_addr
     - bootstrap_kube_vip_addr

bootstrap/templates/addons/csi-driver-nfs/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:
@@ -28,8 +27,6 @@ spec:
   dependsOn:
     - name: local-path-provisioner
       namespace: kube-system
-    - name: ingress-nginx
-      namespace: networking
   values:
     deploymentStrategy:
       type: Recreate
@@ -162,7 +159,7 @@ spec:
       enabled: true
     ingress:
       enabled: true
-      ingressClassName: nginx
+      ingressClassName: internal
       annotations:
         hajimari.io/icon: simple-icons:grafana
       hosts:

bootstrap/templates/addons/grafana/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:
@@ -25,9 +24,6 @@ spec:
       retries: 3
   uninstall:
     keepHistory: false
-  dependsOn:
-    - name: ingress-nginx
-      namespace: networking
   values:
     hajimari:
       title: Apps
@@ -47,7 +43,7 @@ spec:
     ingress:
       main:
         enabled: true
-        ingressClassName: nginx
+        ingressClassName: internal
         annotations:
           hajimari.io/enable: "false"
         hosts:

bootstrap/templates/addons/hajimari/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2

@@ -18,7 +18,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     crds: CreateReplace
     remediation:
       retries: 3
@@ -32,8 +31,6 @@ spec:
   dependsOn:
     - name: local-path-provisioner
       namespace: kube-system
-    - name: ingress-nginx
-      namespace: networking
   values:
     crds:
       enabled: true
@@ -117,7 +114,7 @@ spec:
     prometheus:
       ingress:
         enabled: true
-        ingressClassName: nginx
+        ingressClassName: internal
         annotations:
           hajimari.io/appName: Prometheus
           hajimari.io/icon: simple-icons:prometheus

bootstrap/templates/addons/kube-prometheus-stack/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:
@@ -25,12 +24,7 @@ spec:
       retries: 3
   uninstall:
     keepHistory: false
-  dependsOn:
-    - name: ingress-nginx
-      namespace: networking
   values:
-    env:
-      TZ: "${TIMEZONE}"
     api:
       containers:
         args:

bootstrap/templates/addons/kubernetes-dashboard/ks.yaml.j2

@@ -7,14 +7,13 @@ metadata:
 spec:
   dependsOn:
     - name: cluster-apps-cert-manager
-    - name: cluster-apps-ingress-nginx
     - name: cluster-apps-metrics-server
   path: ./kubernetes/apps/monitoring/kubernetes-dashboard/app
   prune: true
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/system-upgrade-controller/ks.yaml.j2

@@ -28,7 +28,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/weave-gitops/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/addons/weave-gitops/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/ingress.yaml.j2

@@ -5,10 +5,10 @@ metadata:
   name: flux-webhook
   namespace: flux-system
   annotations:
-    external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
+    external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}"
     hajimari.io/enable: "false"
 spec:
-  ingressClassName: nginx
+  ingressClassName: external
   rules:
     - host: &host "flux-webhook.${SECRET_DOMAIN}"
       http:

bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2

@@ -17,7 +17,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2

@@ -17,7 +17,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2

@@ -15,7 +15,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml.j2

@@ -12,7 +12,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/metrics-server/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/reloader/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/networking/cloudflared/app/configs/config.yaml.j2

@@ -2,14 +2,13 @@
 originRequest:
   http2Origin: true
 
-# https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ingress
 ingress:
   - hostname: "${SECRET_DOMAIN}"
-    service: https://ingress-nginx-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.networking.svc.cluster.local:443
     originRequest:
       originServerName: "ingress.${SECRET_DOMAIN}"
   - hostname: "*.${SECRET_DOMAIN}"
-    service: https://ingress-nginx-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.networking.svc.cluster.local:443
     originRequest:
       originServerName: "ingress.${SECRET_DOMAIN}"
   - service: http_status:404