helm install karmada components in order

Signed-off-by: chaosi-zju <[email protected]>
karmada-io · Jun 1, 2024 · a66dcf1 · a66dcf1
1 parent c181917
commit a66dcf1
Show file tree

Hide file tree

Showing 15 changed files with 235 additions and 76 deletions.
diff --git a/charts/karmada/templates/_helpers.tpl b/charts/karmada/templates/_helpers.tpl
@@ -306,6 +306,10 @@ app: {{- include "karmada.name" .}}-search
 {{- include "karmada.commonLabels" . -}}
 {{- end -}}
 
+{{- define "karmada.staticResourceJob.labels" -}}
+{{- include "karmada.commonLabels" . -}}
+{{- end -}}
+
 {{- define "karmada.postInstallJob.labels" -}}
 {{- include "karmada.commonLabels" . -}}
 {{- end -}}
@@ -574,3 +578,59 @@ Return the proper Docker Image Registry Secret Names
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{- define "karmada.init-sa-secret.volume" -}}
+{{- $name := include "karmada.name" . -}}
+- name: init-sa-secret
+  secret:
+    secretName: {{ $name }}-pre-job
+{{- end -}}
+
+{{- define "karmada.init-sa-secret.volumeMount" -}}
+- name: init-sa-secret
+  mountPath: /opt/mount
+{{- end -}}
+
+{{- define "karmada.initContainer.build-kubeconfig" -}}
+TOKEN=$(cat /opt/mount/token)
+kubectl config set-cluster karmada-host --server=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --certificate-authority=/opt/mount/ca.crt
+kubectl config set-credentials default --token=$TOKEN
+kubectl config set-context karmada-host-context --cluster=karmada-host --user=default --namespace=default
+kubectl config use-context karmada-host-context
+{{- end -}}
+
+{{- define "karmada.initContainer.waitETCD" -}}
+{{- if .Values.installInOrder -}}
+- name: wait
+  image: {{ include "karmada.kubectl.image" . }}
+  imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }}
+  command:
+    - /bin/sh
+    - -c
+    - |
+      bash <<'EOF'
+      {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }}
+      kubectl rollout status statefulset etcd -n {{ include "karmada.namespace" . }}
+      EOF
+  volumeMounts:
+    {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }}
+{{- end -}}
+{{- end -}}
+
+{{- define "karmada.initContainer.waitStaticResource" -}}
+{{- if .Values.installInOrder -}}
+- name: wait
+  image: {{ include "karmada.kubectl.image" . }}
+  imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }}
+  command:
+    - /bin/sh
+    - -c
+    - |
+      bash <<'EOF'
+      {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }}
+      kubectl wait --for=condition=complete job {{ include "karmada.name" . }}-static-resource -n {{ include "karmada.namespace" . }}
+      EOF
+  volumeMounts:
+    {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/karmada/templates/karmada-aggregated-apiserver.yaml b/charts/karmada/templates/karmada-aggregated-apiserver.yaml
@@ -29,6 +29,8 @@ spec:
     spec:
       {{- include "karmada.aggregatedApiServer.imagePullSecrets" . | nindent 6 }}
       automountServiceAccountToken: false
+      initContainers:
+        {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}
       containers:
         - name: {{ $name }}-aggregated-apiserver
           image: {{ template "karmada.aggregatedApiServer.image" . }}
@@ -96,6 +98,7 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
+        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:

diff --git a/charts/karmada/templates/karmada-apiserver.yaml b/charts/karmada/templates/karmada-apiserver.yaml
@@ -28,6 +28,8 @@ spec:
     spec:
       {{- include "karmada.apiServer.imagePullSecrets" . | nindent 6 }}
       automountServiceAccountToken: false
+      initContainers:
+        {{- include "karmada.initContainer.waitETCD" . | nindent 8 }}
       containers:
         - name: {{ $name }}-apiserver
           image: {{ template "karmada.apiServer.image" . }}
@@ -135,6 +137,7 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
+      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:
             secretName: {{ $name }}-cert

diff --git a/charts/karmada/templates/karmada-controller-manager.yaml b/charts/karmada/templates/karmada-controller-manager.yaml
@@ -42,7 +42,10 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
+      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
+      initContainers:
+        {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}
       containers:
         - name: {{ $name }}-controller-manager
           image: {{ template "karmada.controllerManager.image" . }}

diff --git a/charts/karmada/templates/karmada-descheduler.yaml b/charts/karmada/templates/karmada-descheduler.yaml
@@ -41,6 +41,8 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       automountServiceAccountToken: false
+      initContainers:
+        {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}
       containers:
         - name: {{ $name }}-descheduler
           image: {{ template "karmada.descheduler.image" . }}
@@ -65,6 +67,7 @@ spec:
           resources:
           {{- toYaml .Values.descheduler.resources | nindent 12 }}
       volumes:
+      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.descheduler.kubeconfig.volume" . | nindent 8 }}
 
 {{ if .Values.descheduler.podDisruptionBudget }}

diff --git a/charts/karmada/templates/karmada-metrics-adapter.yaml b/charts/karmada/templates/karmada-metrics-adapter.yaml
@@ -30,6 +30,8 @@ spec:
     spec:
       {{- include "karmada.metricsAdapter.imagePullSecrets" . | nindent 6 }}
       automountServiceAccountToken: false
+      initContainers:
+        {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}
       containers:
         - name: {{ $name }}-aggregated-apiserver
           image: {{ template "karmada.metricsAdapter.image" . }}
@@ -81,6 +83,7 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
+        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:

diff --git a/charts/karmada/templates/karmada-scheduler.yaml b/charts/karmada/templates/karmada-scheduler.yaml
@@ -41,6 +41,8 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       automountServiceAccountToken: false
+      initContainers:
+        {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}
       containers:
         - name: {{ $name }}-scheduler
           image: {{ template "karmada.scheduler.image" .}}
@@ -65,6 +67,7 @@ spec:
           resources:
           {{- toYaml .Values.scheduler.resources | nindent 12 }}
       volumes:
+      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
 
 {{ if .Values.scheduler.podDisruptionBudget }}

diff --git a/charts/karmada/templates/karmada-search.yaml b/charts/karmada/templates/karmada-search.yaml
@@ -40,6 +40,8 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       automountServiceAccountToken: false
+      initContainers:
+        {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}
       containers:
         - name: {{ $name }}-search
           image: {{ template "karmada.search.image" . }}
@@ -90,6 +92,7 @@ spec:
           resources:
           {{- toYaml .Values.apiServer.resources | nindent 12 }}
       volumes:
+      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.search.kubeconfig.volume" . | nindent 8 }}
       {{- include "karmada.search.etcd.cert.volume" . | nindent 8 }}
 ---

diff --git a/charts/karmada/templates/karmada-static-resource-job.yaml b/charts/karmada/templates/karmada-static-resource-job.yaml
@@ -0,0 +1,104 @@
+{{- $name := include "karmada.name" . -}}
+{{- $namespace := include "karmada.namespace" . -}}
+{{- if eq .Values.installMode "host" }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ $name }}-static-resource"
+  namespace: {{ $namespace }}
+  labels:
+    {{- include "karmada.staticResourceJob.labels" . | nindent 4 }}
+spec:
+  parallelism: 1
+  completions: 1
+  template:
+    metadata:
+      name: {{ $name }}
+      labels:
+        {{- include "karmada.staticResourceJob.labels" . | nindent 8 }}
+    spec:
+      {{- include "karmada.imagePullSecrets" . | nindent 6 }}
+      {{- with .Values.staticResourceJob.tolerations}}
+      tolerations:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.staticResourceJob.nodeSelector }}
+      nodeSelector:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ $name }}-pre-job
+      restartPolicy: Never
+      containers:
+      - name: post-install
+        image: {{ template "karmada.kubectl.image" . }}
+        imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }}
+        command:
+        - /bin/sh
+        - -c
+        - |
+          bash <<'EOF'
+          set -ex
+          kubectl rollout status deployment {{ $name }}-apiserver -n {{ $namespace }}
+          kubectl apply -k /crds --kubeconfig /etc/kubeconfig
+          kubectl apply -f /static-resources --kubeconfig /etc/kubeconfig
+          EOF
+        volumeMounts:
+          - name: {{ $name }}-crds-kustomization
+            mountPath: /crds
+          - name: {{ $name }}-crds-patches
+            mountPath: /crds/patches
+          - name: {{ $name }}-crds-autoscaling-bases
+            mountPath: /crds/bases/autoscaling
+          - name: {{ $name }}-crds-config-bases
+            mountPath: /crds/bases/config
+          - name: {{ $name }}-crds-multicluster-bases
+            mountPath: /crds/bases/multicluster
+          - name: {{ $name }}-crds-networking-bases
+            mountPath: /crds/bases/networking
+          - name: {{ $name }}-crds-policy-bases
+            mountPath: /crds/bases/policy
+          - name: {{ $name }}-crds-remedy-bases
+            mountPath: /crds/bases/remedy
+          - name: {{ $name }}-crds-work-bases
+            mountPath: /crds/bases/work
+          - name: {{ $name }}-crds-apps-bases
+            mountPath: /crds/bases/apps
+          - name: {{ $name }}-static-resources
+            mountPath: /static-resources
+          {{ include "karmada.kubeconfig.volumeMount" . | nindent 10 }}
+      volumes:
+        - name: {{ $name }}-crds-kustomization
+          configMap:
+            name: {{ $name }}-crds-kustomization
+        - name: {{ $name }}-crds-patches
+          configMap:
+            name: {{ $name }}-crds-patches
+        - name: {{ $name }}-crds-autoscaling-bases
+          configMap:
+            name: {{ $name }}-crds-autoscaling-bases
+        - name: {{ $name }}-crds-config-bases
+          configMap:
+            name: {{ $name }}-crds-config-bases
+        - name: {{ $name }}-crds-multicluster-bases
+          configMap:
+            name: {{ $name }}-crds-multicluster-bases
+        - name: {{ $name }}-crds-networking-bases
+          configMap:
+            name: {{ $name }}-crds-networking-bases
+        - name: {{ $name }}-crds-policy-bases
+          configMap:
+            name: {{ $name }}-crds-policy-bases
+        - name: {{ $name }}-crds-remedy-bases
+          configMap:
+            name: {{ $name }}-crds-remedy-bases
+        - name: {{ $name }}-crds-work-bases
+          configMap:
+            name: {{ $name }}-crds-work-bases
+        - name: {{ $name }}-crds-apps-bases
+          configMap:
+            name: {{ $name }}-crds-apps-bases
+        - name: {{ $name }}-static-resources
+          configMap:
+            name: {{ $name }}-static-resources
+        {{ include "karmada.kubeconfig.volume" . | nindent 8 }}
+{{- end }}
diff --git a/charts/karmada/templates/karmada-webhook.yaml b/charts/karmada/templates/karmada-webhook.yaml
@@ -41,6 +41,8 @@ spec:
       tolerations:
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      initContainers:
+        {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}
       containers:
         - name: {{ $name }}-webhook
           image: {{ template "karmada.webhook.image" . }}
@@ -66,6 +68,7 @@ spec:
           resources:
           {{- toYaml .Values.webhook.resources | nindent 12 }}
       volumes:
+      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: {{ $name }}-webhook-cert-secret
           secret:

diff --git a/charts/karmada/templates/kube-controller-manager.yaml b/charts/karmada/templates/kube-controller-manager.yaml
@@ -41,6 +41,8 @@ spec:
       tolerations:
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      initContainers:
+        {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}
       containers:
         - command:
             - kube-controller-manager
@@ -87,6 +89,7 @@ spec:
         - name: apisever-cert
           secret:
             secretName: {{ $name }}-cert
+        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
 
 {{ if .Values.kubeControllerManager.podDisruptionBudget }}

diff --git a/charts/karmada/templates/post-delete-job.yaml b/charts/karmada/templates/post-delete-job.yaml
@@ -47,14 +47,13 @@ spec:
               set -ex
               kubectl delete -f /opt/mount/ --ignore-not-found=true
               kubectl delete -f /opt/crds/ --ignore-not-found=true -R
+              kubectl delete -f /opt/static-resources/ --ignore-not-found=true -R
               kubectl delete cm/{{ $name }}-config -n {{ $namespace }} --ignore-not-found=true
               kubectl delete deployment/{{ $name }}-controller-manager -n {{ $namespace }} --ignore-not-found=true
               EOF
           volumeMounts:
             - name: mount
               mountPath: /opt/mount
-            - name: crds
-              mountPath: /opt/crds
             - name: crds-autoscaling-base
               mountPath: /opt/crds/base/autoscaling
             - name: crds-config-base
@@ -69,35 +68,37 @@ spec:
               mountPath: /opt/crds/base/remedy
             - name: crds-work-base
               mountPath: /opt/crds/base/work
+            - name: static-resources
+              mountPath: /opt/static-resources
       volumes:
       - name: mount
         configMap:
           name: {{ $name }}-config
-      - name: crds
-        configMap:
-          name: {{ $name }}-crds-config
       - name: crds-autoscaling-base
         configMap:
-          name: {{ $name }}-crds-autoscaling-bases-config
+          name: {{ $name }}-crds-autoscaling-bases
       - name: crds-config-base
         configMap:
-          name: {{ $name }}-crds-config-bases-config
+          name: {{ $name }}-crds-config-bases
       - name: crds-multicluster-base
         configMap:
-          name: {{ $name }}-crds-multicluster-bases-config
+          name: {{ $name }}-crds-multicluster-bases
       - name: crds-networking-base
         configMap:
-          name: {{ $name }}-crds-networking-bases-config
+          name: {{ $name }}-crds-networking-bases
       - name: crds-policy-base
         configMap:
-          name: {{ $name }}-crds-policy-bases-config
+          name: {{ $name }}-crds-policy-bases
       - name: crds-remedy-base
         configMap:
-          name: {{ $name }}-crds-remedy-bases-config
+          name: {{ $name }}-crds-remedy-bases
       - name: crds-work-base
         configMap:
-          name: {{ $name }}-crds-work-bases-config
+          name: {{ $name }}-crds-work-bases
       - name: crds-apps-base
         configMap:
-          name: {{ $name }}-crds-apps-bases-config
+          name: {{ $name }}-crds-apps-bases
+      - name: static-resources
+        configMap:
+          name: {{ $name }}-static-resources
 {{- end }}