From 03bc69e46494de989d6635800866f90473bfa7f3 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 31 Jul 2023 10:05:57 -0400 Subject: [PATCH 1/6] feat!: Separate out internal and external nginx controllers Signed-off-by: Devin Buhl --- README.md | 4 +- bootstrap/tasks/validation/net.yaml | 30 ++++-- bootstrap/tasks/validation/vars.yaml | 3 +- .../csi-driver-nfs/app/helmrelease.yaml.j2 | 1 - .../addons/grafana/app/helmrelease.yaml.j2 | 3 +- .../addons/hajimari/app/helmrelease.yaml.j2 | 3 +- .../app/helmrelease.yaml.j2 | 3 +- .../app/helmrelease.yaml.j2 | 1 - .../weave-gitops/app/helmrelease.yaml.j2 | 1 - .../cert-manager/app/helmrelease.yaml.j2 | 1 - .../addons/webhooks/github/ingress.yaml.j2 | 4 +- .../cilium/app/helmrelease.yaml.j2 | 1 - .../coredns/app/helmrelease.yaml.j2 | 1 - .../app/helmrelease.yaml.j2 | 1 - .../metrics-server/app/helmrelease.yaml.j2 | 1 - .../reloader/app/helmrelease.yaml.j2 | 1 - .../cloudflared/app/configs/config.yaml.j2 | 5 +- .../cloudflared/app/dnsendpoint.yaml.j2 | 4 +- .../cloudflared/app/helmrelease.yaml.j2 | 1 - .../echo-server/app/helmrelease.yaml.j2 | 5 +- .../external-dns/app/helmrelease.yaml.j2 | 3 +- .../ingress-nginx/app/helmrelease.yaml.j2 | 76 --------------- .../apps/networking/ingress-nginx/ks.yaml.j2 | 36 ------- .../k8s-gateway/app/helmrelease.yaml.j2 | 1 - .../certificates/kustomization.yaml.j2 | 0 .../certificates/production.yaml.j2 | 0 .../certificates/staging.yaml.j2 | 0 .../nginx/external/helmrelease.yaml.j2 | 97 +++++++++++++++++++ .../external}/kustomization.yaml.j2 | 0 .../nginx/internal/helmrelease.yaml.j2 | 97 +++++++++++++++++++ .../nginx/internal/kustomization.yaml.j2 | 6 ++ .../apps/networking/nginx/ks.yaml.j2 | 55 +++++++++++ bootstrap/vars/config.sample.yaml | 6 +- 33 files changed, 295 insertions(+), 156 deletions(-) delete mode 100644 bootstrap/templates/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml.j2 delete mode 100644 bootstrap/templates/kubernetes/apps/networking/ingress-nginx/ks.yaml.j2 rename bootstrap/templates/kubernetes/apps/networking/{ingress-nginx => nginx}/certificates/kustomization.yaml.j2 (100%) rename bootstrap/templates/kubernetes/apps/networking/{ingress-nginx => nginx}/certificates/production.yaml.j2 (100%) rename bootstrap/templates/kubernetes/apps/networking/{ingress-nginx => nginx}/certificates/staging.yaml.j2 (100%) create mode 100644 bootstrap/templates/kubernetes/apps/networking/nginx/external/helmrelease.yaml.j2 rename bootstrap/templates/kubernetes/apps/networking/{ingress-nginx/app => nginx/external}/kustomization.yaml.j2 (100%) create mode 100644 bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 create mode 100644 bootstrap/templates/kubernetes/apps/networking/nginx/internal/kustomization.yaml.j2 create mode 100644 bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 diff --git a/README.md b/README.md index 9cbb2857f03..1a60dfeef96 100644 --- a/README.md +++ b/README.md @@ -359,7 +359,7 @@ _Mic check, 1, 2_ - In a few moments applications should be lighting up like Chr #### 🌐 DNS -The `external-dns` application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only public sub-domains exposed. In order to make additional applications public you must set an ingress annotation (`external-dns.alpha.kubernetes.io/target`) like done in the `HelmRelease` for `echo-server`. +The `external-dns` application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only public sub-domains exposed. In order to make additional applications public you must set the correct ingress class name and ingress annotations like done in the `HelmRelease` for `echo-server`. For split DNS to work it is required to have `${bootstrap_cloudflare_domain}` point to the `${bootstrap_k8s_gateway_addr}` load balancer IP address on your home DNS server. This will ensure DNS requests for `${bootstrap_cloudflare_domain}` will only get routed to your `k8s_gateway` service thus providing **internal** DNS resolution to your cluster applications/ingresses from any device that uses your home DNS server. @@ -370,7 +370,7 @@ For and example with Pi-Hole apply the following file and restart dnsmasq: server=/${bootstrap_cloudflare_domain}/${bootstrap_k8s_gateway_addr} ``` -Now try to resolve an internal-only domain with `dig @${pi-hole-ip} hajimari.${bootstrap_cloudflare_domain}` it should resolve to your `${bootstrap_ingress_nginx_addr}` IP. +Now try to resolve an internal-only domain with `dig @${pi-hole-ip} hajimari.${bootstrap_cloudflare_domain}` it should resolve to your `${bootstrap_internal_nginx_addr}` IP. If you're having trouble with DNS be sure to check out these two Github discussions, [Internal DNS](https://github.com/onedr0p/flux-cluster-template/discussions/719) and [Pod DNS resolution broken](https://github.com/onedr0p/flux-cluster-template/discussions/635). diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml index 39bb5f38478..116f1c1bbe2 100644 --- a/bootstrap/tasks/validation/net.yaml +++ b/bootstrap/tasks/validation/net.yaml @@ -67,17 +67,29 @@ success_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is within {{ bootstrap_node_cidr }}. fail_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is not within {{ bootstrap_node_cidr }}. -- name: Verify ingress-nginx +- name: Verify internal nginx ansible.builtin.assert: - that: bootstrap_ingress_nginx_addr is ansible.utils.ipv4 - success_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is valid. - fail_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is invalid. + that: bootstrap_internal_nginx_addr is ansible.utils.ipv4 + success_msg: ingress-nginx address {{ bootstrap_internal_nginx_addr }} is valid. + fail_msg: ingress-nginx address {{ bootstrap_internal_nginx_addr }} is invalid. -- name: Verify ingress-nginx in node CIDR +- name: Verify internal nginx in node CIDR ansible.builtin.assert: - that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_ingress_nginx_addr) - success_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is within {{ bootstrap_node_cidr }}. - fail_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is not within {{ bootstrap_node_cidr }}. + that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_internal_nginx_addr) + success_msg: ingress-nginx address {{ bootstrap_internal_nginx_addr }} is within {{ bootstrap_node_cidr }}. + fail_msg: ingress-nginx address {{ bootstrap_internal_nginx_addr }} is not within {{ bootstrap_node_cidr }}. + +- name: Verify external nginx + ansible.builtin.assert: + that: bootstrap_external_nginx_addr is ansible.utils.ipv4 + success_msg: ingress-nginx address {{ bootstrap_external_nginx_addr }} is valid. + fail_msg: ingress-nginx address {{ bootstrap_external_nginx_addr }} is invalid. + +- name: Verify external nginx in node CIDR + ansible.builtin.assert: + that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_external_nginx_addr) + success_msg: ingress-nginx address {{ bootstrap_external_nginx_addr }} is within {{ bootstrap_node_cidr }}. + fail_msg: ingress-nginx address {{ bootstrap_external_nginx_addr }} is not within {{ bootstrap_node_cidr }}. - name: Verify kube-vip ansible.builtin.assert: @@ -93,7 +105,7 @@ - name: Verify nodes are not the same IPs as k8s_gateway, ingress-nginx or kube-vip ansible.builtin.assert: - that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_ingress_nginx_addr, bootstrap_kube_vip_addr) + that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_nginx_addr, bootstrap_internal_nginx_addr, bootstrap_kube_vip_addr) success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip. fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip. quiet: true diff --git a/bootstrap/tasks/validation/vars.yaml b/bootstrap/tasks/validation/vars.yaml index b1d408733ae..97d7ebc3a73 100644 --- a/bootstrap/tasks/validation/vars.yaml +++ b/bootstrap/tasks/validation/vars.yaml @@ -17,7 +17,8 @@ - bootstrap_flux_github_webhook_token - bootstrap_github_repository_name - bootstrap_github_username - - bootstrap_ingress_nginx_addr + - bootstrap_external_nginx_addr + - bootstrap_internal_nginx_addr - bootstrap_ipv6_enabled - bootstrap_k8s_gateway_addr - bootstrap_kube_vip_addr diff --git a/bootstrap/templates/addons/csi-driver-nfs/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/csi-driver-nfs/app/helmrelease.yaml.j2 index 19627f8cf9d..c185cf0e3b4 100644 --- a/bootstrap/templates/addons/csi-driver-nfs/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/csi-driver-nfs/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 index a2b3ce02470..13c979c6bab 100644 --- a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: @@ -162,7 +161,7 @@ spec: enabled: true ingress: enabled: true - ingressClassName: nginx + ingressClassName: internal annotations: hajimari.io/icon: simple-icons:grafana hosts: diff --git a/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 index a06ca044450..5cc59f5b707 100644 --- a/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: @@ -47,7 +46,7 @@ spec: ingress: main: enabled: true - ingressClassName: nginx + ingressClassName: internal annotations: hajimari.io/enable: "false" hosts: diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 index 21a0b20dd45..569d019be07 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 @@ -18,7 +18,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true crds: CreateReplace remediation: retries: 3 @@ -117,7 +116,7 @@ spec: prometheus: ingress: enabled: true - ingressClassName: nginx + ingressClassName: internal annotations: hajimari.io/appName: Prometheus hajimari.io/icon: simple-icons:prometheus diff --git a/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 index ff1892278ce..2c2e1cda823 100644 --- a/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/addons/weave-gitops/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/weave-gitops/app/helmrelease.yaml.j2 index d51dce0d312..27860d61faf 100644 --- a/bootstrap/templates/addons/weave-gitops/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/weave-gitops/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml.j2 index fea46f6e6ed..70b5f2579d5 100644 --- a/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/ingress.yaml.j2 b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/ingress.yaml.j2 index 4857daddf18..c6d00709995 100644 --- a/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/ingress.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/ingress.yaml.j2 @@ -5,10 +5,10 @@ metadata: name: flux-webhook namespace: flux-system annotations: - external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}" + external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}" hajimari.io/enable: "false" spec: - ingressClassName: nginx + ingressClassName: external rules: - host: &host "flux-webhook.${SECRET_DOMAIN}" http: diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 index a84e3f8b398..f9a94b8894a 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 @@ -17,7 +17,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 index 936e5c22e07..ab55980cbc7 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 @@ -17,7 +17,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 index fe363fffab2..be4f0f6fc28 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 @@ -15,7 +15,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml.j2 index ce92031bcd1..b7cf2fa9452 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml.j2 index 22f8afc7b3f..3a38f4eeb9d 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/configs/config.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/configs/config.yaml.j2 index 12b2778ee6a..411794cd3f2 100644 --- a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/configs/config.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/configs/config.yaml.j2 @@ -2,14 +2,13 @@ originRequest: http2Origin: true -# https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ingress ingress: - hostname: "${SECRET_DOMAIN}" - service: https://ingress-nginx-controller.networking.svc.cluster.local:443 + service: https://nginx-external-controller.networking.svc.cluster.local:443 originRequest: originServerName: "ingress.${SECRET_DOMAIN}" - hostname: "*.${SECRET_DOMAIN}" - service: https://ingress-nginx-controller.networking.svc.cluster.local:443 + service: https://nginx-external-controller.networking.svc.cluster.local:443 originRequest: originServerName: "ingress.${SECRET_DOMAIN}" - service: http_status:404 diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml.j2 index 311ee5e219f..2a748f94941 100644 --- a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml.j2 @@ -4,10 +4,8 @@ kind: DNSEndpoint metadata: name: cloudflared namespace: networking - annotations: - external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}" spec: endpoints: - - dnsName: "ingress.${SECRET_DOMAIN}" + - dnsName: "external.${SECRET_DOMAIN}" recordType: CNAME targets: ["${SECRET_CLOUDFLARE_TUNNEL_ID}.cfargotunnel.com"] diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml.j2 index fcb096e4a82..08d27ba1772 100644 --- a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 index d146b3bd519..eca9c40f393 100644 --- a/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 @@ -17,7 +17,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: @@ -58,9 +57,9 @@ spec: ingress: main: enabled: true - ingressClassName: nginx + ingressClassName: external annotations: - external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}" + external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}" hajimari.io/icon: video-input-antenna hosts: - host: &host "{% raw %}{{ .Release.Name }}{% endraw %}.${SECRET_DOMAIN}" diff --git a/bootstrap/templates/kubernetes/apps/networking/external-dns/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/helmrelease.yaml.j2 index 8474475ac83..267cd087c35 100644 --- a/bootstrap/templates/kubernetes/apps/networking/external-dns/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: @@ -35,7 +34,7 @@ spec: name: external-dns-secret key: api-token extraArgs: - - --annotation-filter=external-dns.alpha.kubernetes.io/target + - --ingress-class=external - --cloudflare-proxied - --crd-source-apiversion=externaldns.k8s.io/v1alpha1 - --crd-source-kind=DNSEndpoint diff --git a/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml.j2 deleted file mode 100644 index 9f6f52b16d4..00000000000 --- a/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml.j2 +++ /dev/null @@ -1,76 +0,0 @@ -#jinja2: trim_blocks: True, lstrip_blocks: True ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: ingress-nginx - namespace: networking -spec: - interval: 30m - chart: - spec: - chart: ingress-nginx - version: 4.7.1 - sourceRef: - kind: HelmRepository - name: ingress-nginx - namespace: flux-system - maxHistory: 2 - install: - createNamespace: true - remediation: - retries: 3 - upgrade: - cleanupOnFail: true - remediation: - retries: 3 - uninstall: - keepHistory: false - dependsOn: - - name: cloudflared - namespace: networking - values: - controller: - extraEnvs: - - name: TZ - value: "${TIMEZONE}" - service: - annotations: - external-dns.alpha.kubernetes.io/hostname: "ingress.${SECRET_DOMAIN}" - io.cilium/lb-ipam-ips: "{{ bootstrap_ingress_nginx_addr }}" - externalTrafficPolicy: Cluster - publishService: - enabled: true - ingressClassResource: - default: true - config: - client-header-timeout: 120 - client-body-buffer-size: "100M" - client-body-timeout: 120 - enable-brotli: "true" - hsts-max-age: "31449600" - keep-alive: 120 - keep-alive-requests: 10000 - proxy-body-size: "100M" - ssl-protocols: "TLSv1.3 TLSv1.2" - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: networking - namespaceSelector: - any: true - extraArgs: - {% if bootstrap_acme_production_enabled | default(false) %} - default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls" - {% else %} - default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls" - {% endif %} - resources: - requests: - cpu: 10m - memory: 250Mi - limits: - memory: 500Mi - defaultBackend: - enabled: false diff --git a/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/ks.yaml.j2 deleted file mode 100644 index 00ab322f249..00000000000 --- a/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/ks.yaml.j2 +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: cluster-apps-ingress-nginx-certificates - namespace: flux-system -spec: - dependsOn: - - name: cluster-apps-cert-manager-issuers - path: ./kubernetes/apps/networking/ingress-nginx/certificates - prune: true - sourceRef: - kind: GitRepository - name: home-kubernetes - wait: true - interval: 30m - retryInterval: 1m - timeout: 5m ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: cluster-apps-ingress-nginx - namespace: flux-system -spec: - dependsOn: - - name: cluster-apps-ingress-nginx-certificates - path: ./kubernetes/apps/networking/ingress-nginx/app - prune: true - sourceRef: - kind: GitRepository - name: home-kubernetes - wait: false # no flux ks dependents - interval: 30m - retryInterval: 1m - timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml.j2 index bc6b7eda41e..acef94d21e4 100644 --- a/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml.j2 @@ -16,7 +16,6 @@ spec: namespace: flux-system maxHistory: 2 install: - createNamespace: true remediation: retries: 3 upgrade: diff --git a/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/certificates/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/kustomization.yaml.j2 similarity index 100% rename from bootstrap/templates/kubernetes/apps/networking/ingress-nginx/certificates/kustomization.yaml.j2 rename to bootstrap/templates/kubernetes/apps/networking/nginx/certificates/kustomization.yaml.j2 diff --git a/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/certificates/production.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/production.yaml.j2 similarity index 100% rename from bootstrap/templates/kubernetes/apps/networking/ingress-nginx/certificates/production.yaml.j2 rename to bootstrap/templates/kubernetes/apps/networking/nginx/certificates/production.yaml.j2 diff --git a/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/certificates/staging.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/staging.yaml.j2 similarity index 100% rename from bootstrap/templates/kubernetes/apps/networking/ingress-nginx/certificates/staging.yaml.j2 rename to bootstrap/templates/kubernetes/apps/networking/nginx/certificates/staging.yaml.j2 diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/external/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/external/helmrelease.yaml.j2 new file mode 100644 index 00000000000..9c8749f3c3d --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/external/helmrelease.yaml.j2 @@ -0,0 +1,97 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: nginx-external + namespace: networking +spec: + interval: 30m + chart: + spec: + chart: ingress-nginx + version: 4.7.1 + sourceRef: + kind: HelmRepository + name: ingress-nginx + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: cloudflared + namespace: networking + values: + fullnameOverride: nginx-external + controller: + replicaCount: 1 + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: "external.${SECRET_DOMAIN}" + io.cilium/lb-ipam-ips: "{{ bootstrap_external_nginx_addr }}" + externalTrafficPolicy: Cluster + ingressClassResource: + name: external + default: true + controllerValue: k8s.io/external + admissionWebhooks: + objectSelector: + matchExpressions: + - key: ingress-class + operator: In + values: ["external"] + config: + client-body-buffer-size: 100M + client-body-timeout: 120 + client-header-timeout: 120 + enable-brotli: "true" + enable-real-ip: "true" + hsts-max-age: 31449600 + keep-alive-requests: 10000 + keep-alive: 120 + log-format-escape-json: "true" + log-format-upstream: > + {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for", + "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time, + "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args", + "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer", + "http_user_agent": "$http_user_agent"} + proxy-body-size: 0 + proxy-buffer-size: 16k + ssl-protocols: TLSv1.3 TLSv1.2 + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: networking + namespaceSelector: + any: true + extraArgs: + {% if bootstrap_acme_production_enabled | default(false) %} + default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls" + {% else %} + default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls" + {% endif %} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: nginx-external + app.kubernetes.io/component: controller + resources: + requests: + cpu: 10m + memory: 250Mi + limits: + memory: 500Mi + defaultBackend: + enabled: false diff --git a/bootstrap/templates/kubernetes/apps/networking/ingress-nginx/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/external/kustomization.yaml.j2 similarity index 100% rename from bootstrap/templates/kubernetes/apps/networking/ingress-nginx/app/kustomization.yaml.j2 rename to bootstrap/templates/kubernetes/apps/networking/nginx/external/kustomization.yaml.j2 diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 new file mode 100644 index 00000000000..532fbc5eb49 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 @@ -0,0 +1,97 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: nginx-internal + namespace: networking +spec: + interval: 30m + chart: + spec: + chart: ingress-nginx + version: 4.7.1 + sourceRef: + kind: HelmRepository + name: ingress-nginx + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: cloudflared + namespace: networking + values: + fullnameOverride: nginx-internal + controller: + replicaCount: 1 + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: "internal.${SECRET_DOMAIN}" + io.cilium/lb-ipam-ips: "{{ bootstrap_internal_nginx_addr }}" + externalTrafficPolicy: Cluster + ingressClassResource: + name: internal + default: true + controllerValue: k8s.io/internal + admissionWebhooks: + objectSelector: + matchExpressions: + - key: ingress-class + operator: In + values: ["internal"] + config: + client-body-buffer-size: 100M + client-body-timeout: 120 + client-header-timeout: 120 + enable-brotli: "true" + enable-real-ip: "true" + hsts-max-age: 31449600 + keep-alive-requests: 10000 + keep-alive: 120 + log-format-escape-json: "true" + log-format-upstream: > + {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for", + "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time, + "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args", + "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer", + "http_user_agent": "$http_user_agent"} + proxy-body-size: 0 + proxy-buffer-size: 16k + ssl-protocols: TLSv1.3 TLSv1.2 + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: networking + namespaceSelector: + any: true + extraArgs: + {% if bootstrap_acme_production_enabled | default(false) %} + default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls" + {% else %} + default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls" + {% endif %} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: nginx-internal + app.kubernetes.io/component: controller + resources: + requests: + cpu: 10m + memory: 250Mi + limits: + memory: 500Mi + defaultBackend: + enabled: false diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/internal/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/kustomization.yaml.j2 new file mode 100644 index 00000000000..c83d92a87e5 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: networking +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 new file mode 100644 index 00000000000..91d4c071af2 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 @@ -0,0 +1,55 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-apps-nginx-certificates + namespace: flux-system +spec: + dependsOn: + - name: cluster-apps-cert-manager-issuers + path: ./kubernetes/apps/networking/nginx/certificates + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-apps-nginx-external + namespace: flux-system +spec: + dependsOn: + - name: cluster-apps-external-secrets-stores + - name: cluster-apps-nginx-certificates + path: ./kubernetes/apps/networking/nginx/external + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-apps-nginx-internal + namespace: flux-system +spec: + dependsOn: + - name: cluster-apps-nginx-certificates + path: ./kubernetes/apps/networking/nginx/internal + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index 64a59edead2..14cc5c3d84c 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -40,8 +40,10 @@ bootstrap_node_cidr: bootstrap_kube_vip_addr: # The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used bootstrap_k8s_gateway_addr: -# The Load balancer IP for ingress-nginx, choose an available IP in your nodes network that is not being used -bootstrap_ingress_nginx_addr: +# The Load balancer IP for external nginx, choose an available IP in your nodes network that is not being used +bootstrap_external_nginx_addr: +# The Load balancer IP for internal nginx, choose an available IP in your nodes network that is not being used +bootstrap_internal_nginx_addr: # Keep the next three options default unless you know what you are doing # (Advanced) Enable ipv6 From 6fe7dc709ec6f0f9aaec52b58c98a5d650650c30 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 31 Jul 2023 10:08:59 -0400 Subject: [PATCH 2/6] fix: there is not external secrets here Signed-off-by: Devin Buhl --- bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 index 91d4c071af2..bbbe205f79e 100644 --- a/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 @@ -24,7 +24,6 @@ metadata: namespace: flux-system spec: dependsOn: - - name: cluster-apps-external-secrets-stores - name: cluster-apps-nginx-certificates path: ./kubernetes/apps/networking/nginx/external prune: true From 56adfd3282df2e4934e176fafc0f42efb648fbd8 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 31 Jul 2023 10:10:10 -0400 Subject: [PATCH 3/6] fix: ingress-nginx is not nginx Signed-off-by: Devin Buhl --- .../templates/kubernetes/apps/networking/kustomization.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/templates/kubernetes/apps/networking/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/kustomization.yaml.j2 index e6f8ddc1b83..4ad531cd6e2 100644 --- a/bootstrap/templates/kubernetes/apps/networking/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/kustomization.yaml.j2 @@ -6,5 +6,5 @@ resources: - ./cloudflared/ks.yaml - ./echo-server/ks.yaml - ./external-dns/ks.yaml - - ./ingress-nginx/ks.yaml - ./k8s-gateway/ks.yaml + - ./nginx/ks.yaml From da9fb923aeaa866cfdd40b20e3fccee2d2a45060 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 31 Jul 2023 10:14:32 -0400 Subject: [PATCH 4/6] fix: remove hr ingress-nginx dependencies Signed-off-by: Devin Buhl --- bootstrap/tasks/validation/net.yaml | 16 ++++++++-------- .../addons/grafana/app/helmrelease.yaml.j2 | 2 -- bootstrap/templates/addons/grafana/ks.yaml.j2 | 2 +- .../addons/hajimari/app/helmrelease.yaml.j2 | 3 --- bootstrap/templates/addons/hajimari/ks.yaml.j2 | 2 +- .../app/helmrelease.yaml.j2 | 2 -- .../addons/kube-prometheus-stack/ks.yaml.j2 | 2 +- .../kubernetes-dashboard/app/helmrelease.yaml.j2 | 5 ----- .../addons/kubernetes-dashboard/ks.yaml.j2 | 3 +-- .../addons/system-upgrade-controller/ks.yaml.j2 | 2 +- .../templates/addons/weave-gitops/ks.yaml.j2 | 2 +- .../apps/kube-system/cilium/ks.yaml.j2 | 2 +- .../apps/kube-system/coredns/ks.yaml.j2 | 2 +- .../local-path-provisioner/ks.yaml.j2 | 2 +- .../apps/kube-system/metrics-server/ks.yaml.j2 | 2 +- .../apps/kube-system/reloader/ks.yaml.j2 | 2 +- .../apps/networking/cloudflared/ks.yaml.j2 | 2 +- .../echo-server/app/helmrelease.yaml.j2 | 3 --- .../apps/networking/echo-server/ks.yaml.j2 | 2 +- .../apps/networking/k8s-gateway/ks.yaml.j2 | 2 +- .../kubernetes/apps/networking/nginx/ks.yaml.j2 | 4 ++-- 21 files changed, 24 insertions(+), 40 deletions(-) diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml index 116f1c1bbe2..cb8329685a6 100644 --- a/bootstrap/tasks/validation/net.yaml +++ b/bootstrap/tasks/validation/net.yaml @@ -70,26 +70,26 @@ - name: Verify internal nginx ansible.builtin.assert: that: bootstrap_internal_nginx_addr is ansible.utils.ipv4 - success_msg: ingress-nginx address {{ bootstrap_internal_nginx_addr }} is valid. - fail_msg: ingress-nginx address {{ bootstrap_internal_nginx_addr }} is invalid. + success_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is valid. + fail_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is invalid. - name: Verify internal nginx in node CIDR ansible.builtin.assert: that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_internal_nginx_addr) - success_msg: ingress-nginx address {{ bootstrap_internal_nginx_addr }} is within {{ bootstrap_node_cidr }}. - fail_msg: ingress-nginx address {{ bootstrap_internal_nginx_addr }} is not within {{ bootstrap_node_cidr }}. + success_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is within {{ bootstrap_node_cidr }}. + fail_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is not within {{ bootstrap_node_cidr }}. - name: Verify external nginx ansible.builtin.assert: that: bootstrap_external_nginx_addr is ansible.utils.ipv4 - success_msg: ingress-nginx address {{ bootstrap_external_nginx_addr }} is valid. - fail_msg: ingress-nginx address {{ bootstrap_external_nginx_addr }} is invalid. + success_msg: external nginx address {{ bootstrap_external_nginx_addr }} is valid. + fail_msg: external nginx address {{ bootstrap_external_nginx_addr }} is invalid. - name: Verify external nginx in node CIDR ansible.builtin.assert: that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_external_nginx_addr) - success_msg: ingress-nginx address {{ bootstrap_external_nginx_addr }} is within {{ bootstrap_node_cidr }}. - fail_msg: ingress-nginx address {{ bootstrap_external_nginx_addr }} is not within {{ bootstrap_node_cidr }}. + success_msg: external nginx address {{ bootstrap_external_nginx_addr }} is within {{ bootstrap_node_cidr }}. + fail_msg: external nginx address {{ bootstrap_external_nginx_addr }} is not within {{ bootstrap_node_cidr }}. - name: Verify kube-vip ansible.builtin.assert: diff --git a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 index 13c979c6bab..e7c70e86e32 100644 --- a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 @@ -27,8 +27,6 @@ spec: dependsOn: - name: local-path-provisioner namespace: kube-system - - name: ingress-nginx - namespace: networking values: deploymentStrategy: type: Recreate diff --git a/bootstrap/templates/addons/grafana/ks.yaml.j2 b/bootstrap/templates/addons/grafana/ks.yaml.j2 index 9aeee6f0101..c115caa8383 100644 --- a/bootstrap/templates/addons/grafana/ks.yaml.j2 +++ b/bootstrap/templates/addons/grafana/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 index 5cc59f5b707..d1b4de80766 100644 --- a/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 @@ -24,9 +24,6 @@ spec: retries: 3 uninstall: keepHistory: false - dependsOn: - - name: ingress-nginx - namespace: networking values: hajimari: title: Apps diff --git a/bootstrap/templates/addons/hajimari/ks.yaml.j2 b/bootstrap/templates/addons/hajimari/ks.yaml.j2 index 3123da4929f..1fe9bdeef4b 100644 --- a/bootstrap/templates/addons/hajimari/ks.yaml.j2 +++ b/bootstrap/templates/addons/hajimari/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 index 569d019be07..20890db2a24 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 @@ -31,8 +31,6 @@ spec: dependsOn: - name: local-path-provisioner namespace: kube-system - - name: ingress-nginx - namespace: networking values: crds: enabled: true diff --git a/bootstrap/templates/addons/kube-prometheus-stack/ks.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/ks.yaml.j2 index f7eff255006..74c535889af 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/ks.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 index 2c2e1cda823..d4237a75fa1 100644 --- a/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 @@ -24,12 +24,7 @@ spec: retries: 3 uninstall: keepHistory: false - dependsOn: - - name: ingress-nginx - namespace: networking values: - env: - TZ: "${TIMEZONE}" api: containers: args: diff --git a/bootstrap/templates/addons/kubernetes-dashboard/ks.yaml.j2 b/bootstrap/templates/addons/kubernetes-dashboard/ks.yaml.j2 index bd9152151ce..2220146cffb 100644 --- a/bootstrap/templates/addons/kubernetes-dashboard/ks.yaml.j2 +++ b/bootstrap/templates/addons/kubernetes-dashboard/ks.yaml.j2 @@ -7,14 +7,13 @@ metadata: spec: dependsOn: - name: cluster-apps-cert-manager - - name: cluster-apps-ingress-nginx - name: cluster-apps-metrics-server path: ./kubernetes/apps/monitoring/kubernetes-dashboard/app prune: true sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/addons/system-upgrade-controller/ks.yaml.j2 b/bootstrap/templates/addons/system-upgrade-controller/ks.yaml.j2 index 25d26a34e1d..e4ff61911a8 100644 --- a/bootstrap/templates/addons/system-upgrade-controller/ks.yaml.j2 +++ b/bootstrap/templates/addons/system-upgrade-controller/ks.yaml.j2 @@ -28,7 +28,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/addons/weave-gitops/ks.yaml.j2 b/bootstrap/templates/addons/weave-gitops/ks.yaml.j2 index 7452feebe19..efb4a0c6d8e 100644 --- a/bootstrap/templates/addons/weave-gitops/ks.yaml.j2 +++ b/bootstrap/templates/addons/weave-gitops/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 index 892c4cd36ee..7d29d98212e 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 index ad61e16a8de..c036bad99ca 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml.j2 index 98efc53b013..985be51e775 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml.j2 @@ -12,7 +12,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/ks.yaml.j2 index a81f898da73..d10ca1fbeb1 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/reloader/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/reloader/ks.yaml.j2 index cfa9b764913..27a247c5bf3 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/reloader/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/reloader/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/ks.yaml.j2 index 1dd277b2127..11e3a471113 100644 --- a/bootstrap/templates/kubernetes/apps/networking/cloudflared/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/ks.yaml.j2 @@ -12,7 +12,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 index eca9c40f393..08ac8a0979f 100644 --- a/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 @@ -25,9 +25,6 @@ spec: retries: 3 uninstall: keepHistory: false - dependsOn: - - name: ingress-nginx - namespace: networking values: controller: strategy: RollingUpdate diff --git a/bootstrap/templates/kubernetes/apps/networking/echo-server/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/echo-server/ks.yaml.j2 index 56dd503e71e..0fe3d81a620 100644 --- a/bootstrap/templates/kubernetes/apps/networking/echo-server/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/echo-server/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/ks.yaml.j2 index 6f6ee444a21..502e1f2f82c 100644 --- a/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/ks.yaml.j2 @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: false # no flux ks dependents + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 index bbbe205f79e..60439e48b72 100644 --- a/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 @@ -30,7 +30,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m @@ -48,7 +48,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m From fa322b347948f8fafad6a25d0622fb60f1ac8b42 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 31 Jul 2023 10:15:59 -0400 Subject: [PATCH 5/6] fix: do not need cloudflared for internal nginx Signed-off-by: Devin Buhl --- .../apps/networking/nginx/internal/helmrelease.yaml.j2 | 3 --- 1 file changed, 3 deletions(-) diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 index 532fbc5eb49..fcc391d6de9 100644 --- a/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 @@ -25,9 +25,6 @@ spec: retries: 3 uninstall: keepHistory: false - dependsOn: - - name: cloudflared - namespace: networking values: fullnameOverride: nginx-internal controller: From 683ffb522acc20c084341962917b9bfa5dae1e01 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 31 Jul 2023 13:21:47 -0400 Subject: [PATCH 6/6] fix: add validation for making sure ip addresses are unique Signed-off-by: Devin Buhl --- bootstrap/tasks/validation/net.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml index cb8329685a6..562ce5e2433 100644 --- a/bootstrap/tasks/validation/net.yaml +++ b/bootstrap/tasks/validation/net.yaml @@ -103,7 +103,19 @@ success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}. fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}. -- name: Verify nodes are not the same IPs as k8s_gateway, ingress-nginx or kube-vip +- name: Verify all IP addresses are unique + ansible.builtin.assert: + that: > + [ + bootstrap_k8s_gateway_addr, + bootstrap_external_nginx_addr, + bootstrap_internal_nginx_addr, + bootstrap_kube_vip_addr + ] | unique | length == 4 + success_msg: All IP addresses are unique. + fail_msg: All IP addresses are not unique. + +- name: Verify nodes are not the same IPs as k8s_gateway, nginx external/internal or kube-vip ansible.builtin.assert: that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_nginx_addr, bootstrap_internal_nginx_addr, bootstrap_kube_vip_addr) success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip.