Adding TLS Certificate Authentication to gRPC

Signed-off-by: zhzhuang-zju <[email protected]>

.github/OWNERS

@@ -0,0 +1,7 @@
+reviewers:
+- liangyuanpeng
+- zhzhuang-zju
+
+approvers:
+- liangyuanpeng
+- zhzhuang-zju

.github/dependabot.yml

@@ -11,3 +11,26 @@ updates:
   schedule:
     # Check for updates to GitHub Actions every week
     interval: "weekly"
+
+- package-ecosystem: docker
+  directory: /cluster/images/
+  schedule:
+    interval: weekly
+
+- package-ecosystem: docker
+  directory: /cluster/images/
+  target-branch: "release-1.9"
+  schedule:
+    interval: weekly
+
+- package-ecosystem: docker
+  directory: /cluster/images/
+  target-branch: "release-1.8"
+  schedule:
+    interval: weekly
+
+- package-ecosystem: docker
+  directory: /cluster/images/
+  target-branch: "release-1.7"
+  schedule:
+    interval: weekly

.github/workflows/ci-image-scanning.yaml

@@ -34,15 +34,15 @@ jobs:
           export REGISTRY="docker.io/karmada"
           make image-${{ matrix.target }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.20.0
+        uses: aquasecurity/trivy-action@0.21.0
         with:
           image-ref: 'docker.io/karmada/${{ matrix.target }}:latest'
           format: 'sarif'
           ignore-unfixed: true
           vuln-type: 'os,library'
           output: 'trivy-results.sarif'
       - name: display scan results
-        uses: aquasecurity/trivy-action@0.20.0
+        uses: aquasecurity/trivy-action@0.21.0
         with:
           image-ref: 'docker.io/karmada/${{ matrix.target }}:latest'
           format: 'table'

.github/workflows/ci-schedule-compatibility.yaml

@@ -16,7 +16,7 @@ jobs:
       max-parallel: 5
       fail-fast: false
       matrix:
-        kubeapiserver-version: [ v1.23.4, v1.24.2, v1.25.0, v1.26.0, v1.27.3, v1.28.0, v1.29.0 ]
+        kubeapiserver-version: [ v1.23.4, v1.24.2, v1.25.0, v1.26.0, v1.27.3, v1.28.0, v1.29.0, v1.30.0 ]
         karmada-version: [ release-1.7, release-1.8, release-1.9 ]
     steps:
       # Free up disk space on Ubuntu

.github/workflows/ci-schedule.yml

@@ -16,7 +16,7 @@ jobs:
       max-parallel: 5
       fail-fast: false
       matrix:
-        k8s: [ v1.23.4, v1.24.2, v1.25.0, v1.26.0, v1.27.3, v1.28.0, v1.29.0 ]
+        k8s: [ v1.23.4, v1.24.2, v1.25.0, v1.26.0, v1.27.3, v1.28.0, v1.29.0, v1.30.0 ]
     steps:
       # Free up disk space on Ubuntu
       - name: Free Disk Space (Ubuntu)

.github/workflows/ci.yml

@@ -114,7 +114,7 @@ jobs:
         # Here support the latest three minor releases of Kubernetes, this can be considered to be roughly
         # the same as the End of Life of the Kubernetes release: https://kubernetes.io/releases/
         # Please remember to update the CI Schedule Workflow when we add a new version.
-        k8s: [ v1.27.3, v1.28.0, v1.29.0 ]
+        k8s: [ v1.28.0, v1.29.0, v1.30.0 ]
     steps:
       # Free up disk space on Ubuntu
       - name: Free Disk Space (Ubuntu)

.github/workflows/cli.yaml

@@ -22,7 +22,7 @@ jobs:
         # Here support the latest three minor releases of Kubernetes, this can be considered to be roughly
         # the same as the End of Life of the Kubernetes release: https://kubernetes.io/releases/
         # Please remember to update the CI Schedule Workflow when we add a new version.
-        k8s: [ v1.27.3, v1.28.0, v1.29.0 ]
+        k8s: [ v1.28.0, v1.29.0, v1.30.0 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v4

.go-version

@@ -1 +1 @@
-1.21.10
+1.22.4

.golangci.yml

@@ -82,10 +82,6 @@ linters-settings:
       - name: unexported-return
       - name: time-naming
       - name: empty-block
-  staticcheck:
-    checks:
-      - all
-      - '-SA1019' # disable deprecation check. Tracked by https://github.com/karmada-io/karmada/issues/3835.
 
 issues:
   # The list of ids of default excludes to include or disable. By default it's empty.

OWNERS

@@ -4,6 +4,7 @@ reviewers:
 - jwcesign
 - Poor12
 - RainbowMango
+- whitewindmills
 - XiShanYongYe-Chang
 approvers:
 - chaunceyjiang

README.md

@@ -7,6 +7,7 @@
 [![Releases](https://img.shields.io/github/v/release/karmada-io/karmada)](https://github.com/karmada-io/karmada/releases/latest)
 [![Slack](https://img.shields.io/badge/slack-join-brightgreen)](https://slack.cncf.io)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5301/badge)](https://bestpractices.coreinfrastructure.org/projects/5301)
+[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/karmada-io/karmada/badge)](https://scorecard.dev/viewer/?uri=github.com/karmada-io/karmada)
 ![build](https://github.com/karmada-io/karmada/actions/workflows/ci.yml/badge.svg)
 [![Go Report Card](https://goreportcard.com/badge/github.com/karmada-io/karmada)](https://goreportcard.com/report/github.com/karmada-io/karmada)
 [![codecov](https://codecov.io/gh/karmada-io/karmada/branch/master/graph/badge.svg?token=ROM8CMPXZ6)](https://codecov.io/gh/karmada-io/karmada)
@@ -98,7 +99,7 @@ This guide will cover:
 - Propagate an application by using `karmada`.
 
 ### Prerequisites
-- [Go](https://golang.org/) version v1.21+
+- [Go](https://golang.org/) version v1.22.4+
 - [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version v1.19+
 - [kind](https://kind.sigs.k8s.io/) version v0.14.0+
 

api/openapi-spec/swagger.json

@@ -16759,6 +16759,11 @@
         "workloads"
       ],
       "properties": {
+        "ttlSecondsAfterFinished": {
+          "description": "TTLSecondsAfterFinished limits the lifetime of a WorkloadRebalancer that has finished execution (means each target workload is finished with result of Successful or Failed). If this field is set, ttlSecondsAfterFinished after the WorkloadRebalancer finishes, it is eligible to be automatically deleted. If this field is unset, the WorkloadRebalancer won't be automatically deleted. If this field is set to zero, the WorkloadRebalancer becomes eligible to be deleted immediately after it finishes.",
+          "type": "integer",
+          "format": "int32"
+        },
         "workloads": {
           "description": "Workloads used to specify the list of expected resource. Nil or empty list is not allowed.",
           "type": "array",
@@ -16773,6 +16778,15 @@
       "description": "WorkloadRebalancerStatus contains information about the current status of a WorkloadRebalancer updated periodically by schedule trigger controller.",
       "type": "object",
       "properties": {
+        "finishTime": {
+          "description": "FinishTime represents the finish time of rebalancer.",
+          "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time"
+        },
+        "observedGeneration": {
+          "description": "ObservedGeneration is the generation(.metadata.generation) observed by the controller. If ObservedGeneration is less than the generation in metadata means the controller hasn't confirmed the rebalance result or hasn't done the rebalance yet.",
+          "type": "integer",
+          "format": "int64"
+        },
         "observedWorkloads": {
           "description": "ObservedWorkloads contains information about the execution states and messages of target resources.",
           "type": "array",
@@ -17591,7 +17605,7 @@
       ],
       "properties": {
         "luaScript": {
-          "description": "LuaScript holds the Lua script that is used to interpret the dependencies of a specific resource. The script should implement a function as follows:\n\n```\n  luaScript: \u003e\n      function GetDependencies(desiredObj)\n          dependencies = {}\n          if desiredObj.spec.serviceAccountName ~= nil and desiredObj.spec.serviceAccountName ~= \"default\" then\n              dependency = {}\n              dependency.apiVersion = \"v1\"\n              dependency.kind = \"ServiceAccount\"\n              dependency.name = desiredObj.spec.serviceAccountName\n              dependency.namespace = desiredObj.namespace\n              dependencies[1] = {}\n              dependencies[1] = dependency\n          end\n          return dependencies\n      end\n```\n\nThe content of the LuaScript needs to be a whole function including both declaration and implementation.\n\nThe parameters will be supplied by the system:\n  - desiredObj: the object represents the configuration to be applied\n      to the member cluster.\n\nThe returned value should be expressed by a slice of DependentObjectReference.",
+          "description": "LuaScript holds the Lua script that is used to interpret the dependencies of a specific resource. The script should implement a function as follows:\n\n```\n  luaScript: \u003e\n      function GetDependencies(desiredObj)\n          dependencies = {}\n          serviceAccountName = desiredObj.spec.template.spec.serviceAccountName\n          if serviceAccountName ~= nil and serviceAccountName ~= \"default\" then\n              dependency = {}\n              dependency.apiVersion = \"v1\"\n              dependency.kind = \"ServiceAccount\"\n              dependency.name = serviceAccountName\n              dependency.namespace = desiredObj.metadata.namespace\n              dependencies[1] = dependency\n          end\n          return dependencies\n      end\n```\n\nThe content of the LuaScript needs to be a whole function including both declaration and implementation.\n\nThe parameters will be supplied by the system:\n  - desiredObj: the object represents the configuration to be applied\n      to the member cluster.\n\nThe returned value should be expressed by a slice of DependentObjectReference.",
           "type": "string",
           "default": ""
         }

artifacts/deploy/karmada-descheduler.yaml

@@ -27,6 +27,9 @@ spec:
             - /bin/karmada-descheduler
             - --kubeconfig=/etc/kubeconfig
             - --bind-address=0.0.0.0
+            - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
+            - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
+            - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
             - --v=4
           livenessProbe:
             httpGet:
@@ -38,10 +41,16 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 5
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: kubeconfig
               subPath: kubeconfig
               mountPath: /etc/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
         - name: kubeconfig
           secret:
             secretName: kubeconfig

artifacts/deploy/karmada-scheduler-estimator.yaml

@@ -27,6 +27,9 @@ spec:
             - /bin/karmada-scheduler-estimator
             - --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig
             - --cluster-name={{member_cluster_name}}
+            - --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt
+            - --grpc-auth-key-file=/etc/karmada/pki/karmada.key
+            - --grpc-client-ca-file=/etc/karmada/pki/ca.crt
           livenessProbe:
             httpGet:
               path: /healthz
@@ -37,10 +40,16 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 5
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: member-kubeconfig
               subPath: {{member_cluster_name}}-kubeconfig
               mountPath: /etc/{{member_cluster_name}}-kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
         - name: member-kubeconfig
           secret:
             secretName: {{member_cluster_name}}-kubeconfig

artifacts/deploy/karmada-scheduler.yaml

@@ -38,12 +38,21 @@ spec:
             - --bind-address=0.0.0.0
             - --secure-port=10351
             - --enable-scheduler-estimator=true
+            - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
+            - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
+            - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
             - --v=4
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: kubeconfig
               subPath: kubeconfig
               mountPath: /etc/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
         - name: kubeconfig
           secret:
             secretName: kubeconfig

charts/karmada/_crds/bases/apps/apps.karmada.io_workloadrebalancers.yaml

@@ -41,6 +41,15 @@ spec:
             description: Spec represents the specification of the desired behavior
               of WorkloadRebalancer.
             properties:
+              ttlSecondsAfterFinished:
+                description: |-
+                  TTLSecondsAfterFinished limits the lifetime of a WorkloadRebalancer that has finished execution (means each
+                  target workload is finished with result of Successful or Failed).
+                  If this field is set, ttlSecondsAfterFinished after the WorkloadRebalancer finishes, it is eligible to be automatically deleted.
+                  If this field is unset, the WorkloadRebalancer won't be automatically deleted.
+                  If this field is set to zero, the WorkloadRebalancer becomes eligible to be deleted immediately after it finishes.
+                format: int32
+                type: integer
               workloads:
                 description: |-
                   Workloads used to specify the list of expected resource.
@@ -76,6 +85,17 @@ spec:
           status:
             description: Status represents the status of WorkloadRebalancer.
             properties:
+              finishTime:
+                description: FinishTime represents the finish time of rebalancer.
+                format: date-time
+                type: string
+              observedGeneration:
+                description: |-
+                  ObservedGeneration is the generation(.metadata.generation) observed by the controller.
+                  If ObservedGeneration is less than the generation in metadata means the controller hasn't confirmed
+                  the rebalance result or hasn't done the rebalance yet.
+                format: int64
+                type: integer
               observedWorkloads:
                 description: ObservedWorkloads contains information about the execution
                   states and messages of target resources.

charts/karmada/_crds/bases/config/config.karmada.io_resourceinterpretercustomizations.yaml

@@ -69,13 +69,13 @@ spec:
                             luaScript: >
                                 function GetDependencies(desiredObj)
                                     dependencies = {}
-                                    if desiredObj.spec.serviceAccountName ~= nil and desiredObj.spec.serviceAccountName ~= "default" then
+                                    serviceAccountName = desiredObj.spec.template.spec.serviceAccountName
+                                    if serviceAccountName ~= nil and serviceAccountName ~= "default" then
                                         dependency = {}
                                         dependency.apiVersion = "v1"
                                         dependency.kind = "ServiceAccount"
-                                        dependency.name = desiredObj.spec.serviceAccountName
-                                        dependency.namespace = desiredObj.namespace
-                                        dependencies[1] = {}
+                                        dependency.name = serviceAccountName
+                                        dependency.namespace = desiredObj.metadata.namespace
                                         dependencies[1] = dependency
                                     end
                                     return dependencies

charts/karmada/templates/_helpers.tpl

@@ -306,6 +306,10 @@ app: {{- include "karmada.name" .}}-search
 {{- include "karmada.commonLabels" . -}}
 {{- end -}}
 
+{{- define "karmada.staticResourceJob.labels" -}}
+{{- include "karmada.commonLabels" . -}}
+{{- end -}}
+
 {{- define "karmada.postInstallJob.labels" -}}
 {{- include "karmada.commonLabels" . -}}
 {{- end -}}
@@ -574,3 +578,55 @@ Return the proper Docker Image Registry Secret Names
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{- define "karmada.init-sa-secret.volume" -}}
+{{- $name := include "karmada.name" . -}}
+- name: init-sa-secret
+  secret:
+    secretName: {{ $name }}-hook-job
+{{- end -}}
+
+{{- define "karmada.init-sa-secret.volumeMount" -}}
+- name: init-sa-secret
+  mountPath: /opt/mount
+{{- end -}}
+
+{{- define "karmada.initContainer.build-kubeconfig" -}}
+TOKEN=$(cat /opt/mount/token)
+kubectl config set-cluster karmada-host --server=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --certificate-authority=/opt/mount/ca.crt
+kubectl config set-credentials default --token=$TOKEN
+kubectl config set-context karmada-host-context --cluster=karmada-host --user=default --namespace=default
+kubectl config use-context karmada-host-context
+{{- end -}}
+
+{{- define "karmada.initContainer.waitEtcd" -}}
+- name: wait
+  image: {{ include "karmada.kubectl.image" . }}
+  imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }}
+  command:
+    - /bin/sh
+    - -c
+    - |
+      bash <<'EOF'
+      {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }}
+      kubectl rollout status statefulset etcd -n {{ include "karmada.namespace" . }}
+      EOF
+  volumeMounts:
+    {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }}
+{{- end -}}
+
+{{- define "karmada.initContainer.waitStaticResource" -}}
+- name: wait
+  image: {{ include "karmada.kubectl.image" . }}
+  imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }}
+  command:
+    - /bin/sh
+    - -c
+    - |
+      bash <<'EOF'
+      {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }}
+      kubectl wait --for=condition=complete job {{ include "karmada.name" . }}-static-resource -n {{ include "karmada.namespace" . }}
+      EOF
+  volumeMounts:
+    {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }}
+{{- end -}}

charts/karmada/templates/karmada-aggregated-apiserver.yaml

@@ -29,6 +29,8 @@ spec:
     spec:
       {{- include "karmada.aggregatedApiServer.imagePullSecrets" . | nindent 6 }}
       automountServiceAccountToken: false
+      initContainers:
+        {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}
       containers:
         - name: {{ $name }}-aggregated-apiserver
           image: {{ template "karmada.aggregatedApiServer.image" . }}
@@ -96,6 +98,7 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
+        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:

charts/karmada/templates/karmada-apiserver.yaml

@@ -28,6 +28,8 @@ spec:
     spec:
       {{- include "karmada.apiServer.imagePullSecrets" . | nindent 6 }}
       automountServiceAccountToken: false
+      initContainers:
+        {{- include "karmada.initContainer.waitEtcd" . | nindent 8 }}
       containers:
         - name: {{ $name }}-apiserver
           image: {{ template "karmada.apiServer.image" . }}
@@ -135,6 +137,7 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
+      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:
             secretName: {{ $name }}-cert