From edb224d7d22e2fb4dc904d6cb01dddc071a78095 Mon Sep 17 00:00:00 2001 From: chaosi-zju Date: Wed, 16 Oct 2024 11:45:03 +0800 Subject: [PATCH] standardize the naming of karmada secrets in local up method Signed-off-by: chaosi-zju --- .../deploy/karmada-aggregated-apiserver.yaml | 24 ++-- artifacts/deploy/karmada-apiserver.yaml | 52 ++++++--- ...ecret.yaml => karmada-ca-cert-secret.yaml} | 6 +- artifacts/deploy/karmada-cert-secret.yaml | 36 ++---- artifacts/deploy/karmada-descheduler.yaml | 14 +-- artifacts/deploy/karmada-etcd.yaml | 31 ++--- artifacts/deploy/karmada-key-pair-secret.yaml | 11 ++ artifacts/deploy/karmada-metrics-adapter.yaml | 14 +-- .../deploy/karmada-scheduler-estimator.yaml | 14 +-- artifacts/deploy/karmada-scheduler.yaml | 14 +-- artifacts/deploy/karmada-search.yaml | 24 ++-- artifacts/deploy/karmada-webhook.yaml | 10 +- artifacts/deploy/kube-controller-manager.yaml | 27 +++-- .../karmada-interpreter-webhook-example.yaml | 10 +- hack/deploy-karmada.sh | 108 +++++++++++------- hack/util.sh | 12 ++ 16 files changed, 236 insertions(+), 171 deletions(-) rename artifacts/deploy/{karmada-webhook-cert-secret.yaml => karmada-ca-cert-secret.yaml} (64%) create mode 100644 artifacts/deploy/karmada-key-pair-secret.yaml diff --git a/artifacts/deploy/karmada-aggregated-apiserver.yaml b/artifacts/deploy/karmada-aggregated-apiserver.yaml index 96e85e76b503..00e651f723ce 100644 --- a/artifacts/deploy/karmada-aggregated-apiserver.yaml +++ b/artifacts/deploy/karmada-aggregated-apiserver.yaml @@ -30,11 +30,11 @@ spec: - --authentication-kubeconfig=/etc/karmada/config/karmada.config - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379 - - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt - - --etcd-certfile=/etc/karmada/pki/etcd-client.crt - - --etcd-keyfile=/etc/karmada/pki/etcd-client.key - - --tls-cert-file=/etc/karmada/pki/karmada.crt - - --tls-private-key-file=/etc/karmada/pki/karmada.key + - --etcd-cafile=/etc/karmada/pki/etcd-client/ca.crt + - --etcd-certfile=/etc/karmada/pki/etcd-client/tls.crt + - --etcd-keyfile=/etc/karmada/pki/etcd-client/tls.key + - --tls-cert-file=/etc/karmada/pki/server/tls.crt + - --tls-private-key-file=/etc/karmada/pki//server/tls.key - --audit-log-path=- - --audit-log-maxage=0 - --audit-log-maxbackup=0 @@ -61,16 +61,22 @@ spec: volumeMounts: - name: karmada-config mountPath: /etc/karmada/config - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: server-cert + mountPath: /etc/karmada/pki/server + readOnly: true + - name: etcd-client-cert + mountPath: /etc/karmada/pki/etcd-client readOnly: true volumes: - name: karmada-config secret: secretName: karmada-aggregated-apiserver-config - - name: karmada-certs + - name: server-cert + secret: + secretName: karmada-aggregated-apiserver-cert + - name: etcd-client-cert secret: - secretName: karmada-cert-secret + secretName: karmada-aggregated-apiserver-etcd-client-cert --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/karmada-apiserver.yaml b/artifacts/deploy/karmada-apiserver.yaml index 5d5c2f458b5f..ef26ae3116b0 100644 --- a/artifacts/deploy/karmada-apiserver.yaml +++ b/artifacts/deploy/karmada-apiserver.yaml @@ -36,29 +36,29 @@ spec: - kube-apiserver - --allow-privileged=true - --authorization-mode=Node,RBAC - - --client-ca-file=/etc/karmada/pki/ca.crt - --enable-bootstrap-token-auth=true - - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt - - --etcd-certfile=/etc/karmada/pki/etcd-client.crt - - --etcd-keyfile=/etc/karmada/pki/etcd-client.key + - --etcd-cafile=/etc/karmada/pki/etcd-client/ca.crt + - --etcd-certfile=/etc/karmada/pki/etcd-client/tls.crt + - --etcd-keyfile=/etc/karmada/pki/etcd-client/tls.key - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379 - --bind-address=0.0.0.0 - --disable-admission-plugins=StorageObjectInUseProtection,ServiceAccount - --runtime-config= - --secure-port=5443 - --service-account-issuer=https://kubernetes.default.svc.cluster.local - - --service-account-key-file=/etc/karmada/pki/karmada.key - - --service-account-signing-key-file=/etc/karmada/pki/karmada.key + - --service-account-key-file=/etc/karmada/pki/service-account-key-pair/sa.pub + - --service-account-signing-key-file=/etc/karmada/pki/service-account-key-pair/sa.key - --service-cluster-ip-range=10.96.0.0/12 - - --proxy-client-cert-file=/etc/karmada/pki/front-proxy-client.crt - - --proxy-client-key-file=/etc/karmada/pki/front-proxy-client.key + - --proxy-client-cert-file=/etc/karmada/pki/front-proxy-client/tls.crt + - --proxy-client-key-file=/etc/karmada/pki/front-proxy-client/tls.key + - --requestheader-client-ca-file=/etc/karmada/pki/front-proxy-client/ca.crt - --requestheader-allowed-names=front-proxy-client - - --requestheader-client-ca-file=/etc/karmada/pki/front-proxy-ca.crt - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - - --tls-cert-file=/etc/karmada/pki/apiserver.crt - - --tls-private-key-file=/etc/karmada/pki/apiserver.key + - --tls-cert-file=/etc/karmada/pki/server/tls.crt + - --tls-private-key-file=/etc/karmada/pki/server/tls.key + - --client-ca-file=/etc/karmada/pki/server/ca.crt - --tls-min-version=VersionTLS13 name: karmada-apiserver image: registry.k8s.io/kube-apiserver:{{karmada_apiserver_version}} @@ -88,9 +88,31 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /etc/karmada/pki - name: karmada-certs + - name: server-cert + mountPath: /etc/karmada/pki/server readOnly: true + - name: etcd-client-cert + mountPath: /etc/karmada/pki/etcd-client + readOnly: true + - name: front-proxy-client-cert + mountPath: /etc/karmada/pki/front-proxy-client + readOnly: true + - name: service-account-key-pair + mountPath: /etc/karmada/pki/service-account-key-pair + readOnly: true + volumes: + - name: server-cert + secret: + secretName: karmada-apiserver-cert + - name: etcd-client-cert + secret: + secretName: karmada-apiserver-etcd-client-cert + - name: front-proxy-client-cert + secret: + secretName: karmada-apiserver-front-proxy-client-cert + - name: service-account-key-pair + secret: + secretName: karmada-apiserver-service-account-key-pair dnsPolicy: ClusterFirstWithHostNet enableServiceLinks: true hostNetwork: true @@ -104,10 +126,6 @@ spec: tolerations: - effect: NoExecute operator: Exists - volumes: - - name: karmada-certs - secret: - secretName: karmada-cert-secret --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/karmada-webhook-cert-secret.yaml b/artifacts/deploy/karmada-ca-cert-secret.yaml similarity index 64% rename from artifacts/deploy/karmada-webhook-cert-secret.yaml rename to artifacts/deploy/karmada-ca-cert-secret.yaml index aabdeedc2ef2..afe566040e0e 100644 --- a/artifacts/deploy/karmada-webhook-cert-secret.yaml +++ b/artifacts/deploy/karmada-ca-cert-secret.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: Secret metadata: - name: webhook-cert + name: ${component}-ca-cert namespace: karmada-system type: kubernetes.io/tls data: tls.crt: | - {{server_certificate}} + ${ca_crt} tls.key: | - {{server_key}} + ${ca_key} diff --git a/artifacts/deploy/karmada-cert-secret.yaml b/artifacts/deploy/karmada-cert-secret.yaml index 2a32ae4b2eb5..344a65fe14f8 100644 --- a/artifacts/deploy/karmada-cert-secret.yaml +++ b/artifacts/deploy/karmada-cert-secret.yaml @@ -1,35 +1,13 @@ apiVersion: v1 kind: Secret metadata: - name: karmada-cert-secret + name: ${name}-cert namespace: karmada-system -type: Opaque +type: kubernetes.io/tls data: ca.crt: | - {{ca_crt}} - ca.key: | - {{ca_key}} - karmada.crt: | - {{client_crt}} - karmada.key: | - {{client_key}} - apiserver.crt: | - {{apiserver_crt}} - apiserver.key: | - {{apiserver_key}} - front-proxy-ca.crt: | - {{front_proxy_ca_crt}} - front-proxy-client.crt: | - {{front_proxy_client_crt}} - front-proxy-client.key: | - {{front_proxy_client_key}} - etcd-ca.crt: | - {{etcd_ca_crt}} - etcd-server.crt: | - {{etcd_server_crt}} - etcd-server.key: | - {{etcd_server_key}} - etcd-client.crt: | - {{etcd_client_crt}} - etcd-client.key: | - {{etcd_client_key}} + ${ca_crt} + tls.crt: | + ${tls_crt} + tls.key: | + ${tls_key} diff --git a/artifacts/deploy/karmada-descheduler.yaml b/artifacts/deploy/karmada-descheduler.yaml index 1b10b0a25def..46a0f48516be 100644 --- a/artifacts/deploy/karmada-descheduler.yaml +++ b/artifacts/deploy/karmada-descheduler.yaml @@ -28,9 +28,9 @@ spec: - --kubeconfig=/etc/karmada/config/karmada.config - --metrics-bind-address=0.0.0.0:8080 - --health-probe-bind-address=0.0.0.0:10358 - - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt - - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt - - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key + - --scheduler-estimator-ca-file=/etc/karmada/pki/scheduler-estimator-client/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/scheduler-estimator-client/tls.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/scheduler-estimator-client/tls.key - --v=4 livenessProbe: httpGet: @@ -48,13 +48,13 @@ spec: volumeMounts: - name: karmada-config mountPath: /etc/karmada/config - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: scheduler-estimator-client-cert + mountPath: /etc/karmada/pki/scheduler-estimator-client readOnly: true volumes: - name: karmada-config secret: secretName: karmada-descheduler-config - - name: karmada-certs + - name: scheduler-estimator-client-cert secret: - secretName: karmada-cert-secret + secretName: karmada-descheduler-scheduler-estimator-client-cert diff --git a/artifacts/deploy/karmada-etcd.yaml b/artifacts/deploy/karmada-etcd.yaml index d429700b0ebf..d2f135452837 100644 --- a/artifacts/deploy/karmada-etcd.yaml +++ b/artifacts/deploy/karmada-etcd.yaml @@ -40,7 +40,7 @@ spec: command: - /bin/sh - -ec - - 'etcdctl get /registry --prefix --keys-only --endpoints https://127.0.0.1:2379 --cacert /etc/karmada/pki/etcd-ca.crt --cert /etc/karmada/pki/etcd-server.crt --key /etc/karmada/pki/etcd-server.key' + - 'etcdctl get /registry --prefix --keys-only --endpoints https://127.0.0.1:2379 --cacert /etc/karmada/pki/etcd-client/ca.crt --cert /etc/karmada/pki/etcd-client/tls.crt --key /etc/karmada/pki/etcd-client/tls.key' failureThreshold: 3 initialDelaySeconds: 600 periodSeconds: 60 @@ -53,11 +53,6 @@ spec: - containerPort: 2380 name: server protocol: TCP - volumeMounts: - - mountPath: /var/lib/etcd - name: etcd-data - - mountPath: /etc/karmada/pki - name: etcd-certs resources: requests: cpu: 100m @@ -76,24 +71,34 @@ spec: - etcd0=http://etcd-0.etcd.karmada-system.svc.cluster.local:2380 - --initial-cluster-state - new - - --cert-file=/etc/karmada/pki/etcd-server.crt - --client-cert-auth=true - - --key-file=/etc/karmada/pki/etcd-server.key - - --trusted-ca-file=/etc/karmada/pki/etcd-ca.crt + - --cert-file=/etc/karmada/pki/server/tls.crt + - --key-file=/etc/karmada/pki/server/tls.key + - --trusted-ca-file=/etc/karmada/pki/server/ca.crt - --data-dir=/var/lib/etcd - --snapshot-count=10000 # Setting Golang's secure cipher suites as etcd's cipher suites. # They are obtained by the return value of the function CipherSuites() under the go/src/crypto/tls/cipher_suites.go package. # Consistent with the Preferred values of k8s’s default cipher suites. - --cipher-suites=TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + volumeMounts: + - name: etcd-data + mountPath: /var/lib/etcd + - name: server-cert + mountPath: /etc/karmada/pki/server + - name: etcd-client-cert + mountPath: /etc/karmada/pki/etcd-client volumes: - - hostPath: + - name: etcd-data + hostPath: path: /var/lib/karmada-etcd type: DirectoryOrCreate - name: etcd-data - - name: etcd-certs + - name: server-cert + secret: + secretName: etcd-cert + - name: etcd-client-cert secret: - secretName: karmada-cert-secret + secretName: etcd-etcd-client-cert --- apiVersion: v1 diff --git a/artifacts/deploy/karmada-key-pair-secret.yaml b/artifacts/deploy/karmada-key-pair-secret.yaml new file mode 100644 index 000000000000..58bb203f168f --- /dev/null +++ b/artifacts/deploy/karmada-key-pair-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ${component}-service-account-key-pair + namespace: karmada-system +type: Opaque +data: + sa.pub: | + ${sa_pub} + sa.key: | + ${sa_key} diff --git a/artifacts/deploy/karmada-metrics-adapter.yaml b/artifacts/deploy/karmada-metrics-adapter.yaml index 8405c057b647..678edb13baf5 100644 --- a/artifacts/deploy/karmada-metrics-adapter.yaml +++ b/artifacts/deploy/karmada-metrics-adapter.yaml @@ -29,9 +29,9 @@ spec: - --kubeconfig=/etc/karmada/config/karmada.config - --authentication-kubeconfig=/etc/karmada/config/karmada.config - --authorization-kubeconfig=/etc/karmada/config/karmada.config - - --client-ca-file=/etc/karmada/pki/ca.crt - - --tls-cert-file=/etc/karmada/pki/karmada.crt - - --tls-private-key-file=/etc/karmada/pki/karmada.key + - --client-ca-file=/etc/karmada/pki/server/ca.crt + - --tls-cert-file=/etc/karmada/pki/server/tls.crt + - --tls-private-key-file=/etc/karmada/pki/server/tls.key - --audit-log-path=- - --audit-log-maxage=0 - --audit-log-maxbackup=0 @@ -60,16 +60,16 @@ spec: volumeMounts: - name: karmada-config mountPath: /etc/karmada/config - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: server-cert + mountPath: /etc/karmada/pki/server readOnly: true volumes: - name: karmada-config secret: secretName: karmada-metrics-adapter-config - - name: karmada-certs + - name: server-cert secret: - secretName: karmada-cert-secret + secretName: karmada-metrics-adapter-cert --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/karmada-scheduler-estimator.yaml b/artifacts/deploy/karmada-scheduler-estimator.yaml index b1ed2a3abaed..e44ef8c3a216 100644 --- a/artifacts/deploy/karmada-scheduler-estimator.yaml +++ b/artifacts/deploy/karmada-scheduler-estimator.yaml @@ -27,9 +27,9 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig - --cluster-name={{member_cluster_name}} - - --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt - - --grpc-auth-key-file=/etc/karmada/pki/karmada.key - - --grpc-client-ca-file=/etc/karmada/pki/ca.crt + - --grpc-auth-cert-file=/etc/karmada/pki/server/tls.crt + - --grpc-auth-key-file=/etc/karmada/pki/server/tls.key + - --grpc-client-ca-file=/etc/karmada/pki/server/ca.crt - --metrics-bind-address=0.0.0.0:8080 - --health-probe-bind-address=0.0.0.0:10351 livenessProbe: @@ -46,16 +46,16 @@ spec: name: metrics protocol: TCP volumeMounts: - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: server-cert + mountPath: /etc/karmada/pki/server readOnly: true - name: member-kubeconfig subPath: {{member_cluster_name}}-kubeconfig mountPath: /etc/{{member_cluster_name}}-kubeconfig volumes: - - name: karmada-certs + - name: server-cert secret: - secretName: karmada-cert-secret + secretName: karmada-metrics-adapter-cert - name: member-kubeconfig secret: secretName: {{member_cluster_name}}-kubeconfig diff --git a/artifacts/deploy/karmada-scheduler.yaml b/artifacts/deploy/karmada-scheduler.yaml index 8ea2a933499b..1604b9c9e1d3 100644 --- a/artifacts/deploy/karmada-scheduler.yaml +++ b/artifacts/deploy/karmada-scheduler.yaml @@ -42,20 +42,20 @@ spec: - --metrics-bind-address=0.0.0.0:8080 - --health-probe-bind-address=0.0.0.0:10351 - --enable-scheduler-estimator=true - - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt - - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt - - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key + - --scheduler-estimator-ca-file=/etc/karmada/pki/scheduler-estimator-client/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/scheduler-estimator-client/tls.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/scheduler-estimator-client/tls.key - --v=4 volumeMounts: - name: karmada-config mountPath: /etc/karmada/config - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: scheduler-estimator-client-cert + mountPath: /etc/karmada/pki/scheduler-estimator-client readOnly: true volumes: - name: karmada-config secret: secretName: karmada-scheduler-config - - name: karmada-certs + - name: scheduler-estimator-client-cert secret: - secretName: karmada-cert-secret + secretName: karmada-scheduler-scheduler-estimator-client-cert diff --git a/artifacts/deploy/karmada-search.yaml b/artifacts/deploy/karmada-search.yaml index 068030d5cbc1..5c18e788bb0f 100644 --- a/artifacts/deploy/karmada-search.yaml +++ b/artifacts/deploy/karmada-search.yaml @@ -30,11 +30,11 @@ spec: - --authentication-kubeconfig=/etc/karmada/config/karmada.config - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379 - - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt - - --etcd-certfile=/etc/karmada/pki/etcd-client.crt - - --etcd-keyfile=/etc/karmada/pki/etcd-client.key - - --tls-cert-file=/etc/karmada/pki/karmada.crt - - --tls-private-key-file=/etc/karmada/pki/karmada.key + - --etcd-cafile=/etc/karmada/pki/etcd-client/ca.crt + - --etcd-certfile=/etc/karmada/pki/etcd-client/tls.crt + - --etcd-keyfile=/etc/karmada/pki/etcd-client/tls.key + - --tls-cert-file=/etc/karmada/pki/server/tls.crt + - --tls-private-key-file=/etc/karmada/pki/server/tls.key - --audit-log-path=- - --audit-log-maxage=0 - --audit-log-maxbackup=0 @@ -54,16 +54,22 @@ spec: volumeMounts: - name: karmada-config mountPath: /etc/karmada/config - - name: karmada-certs - mountPath: /etc/karmada/pki + - name: server-cert + mountPath: /etc/karmada/pki/server + readOnly: true + - name: etcd-client-cert + mountPath: /etc/karmada/pki/etcd-client readOnly: true volumes: - name: karmada-config secret: secretName: karmada-search-config - - name: karmada-certs + - name: server-cert + secret: + secretName: karmada-search-cert + - name: etcd-client-cert secret: - secretName: karmada-cert-secret + secretName: karmada-search-etcd-client-cert --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/karmada-webhook.yaml b/artifacts/deploy/karmada-webhook.yaml index a460f429cacb..850bdc5b2142 100644 --- a/artifacts/deploy/karmada-webhook.yaml +++ b/artifacts/deploy/karmada-webhook.yaml @@ -31,7 +31,7 @@ spec: - --default-not-ready-toleration-seconds=30 - --default-unreachable-toleration-seconds=30 - --secure-port=8443 - - --cert-dir=/var/serving-cert + - --cert-dir=/etc/karmada/pki/server - --v=4 ports: - containerPort: 8443 @@ -46,16 +46,16 @@ spec: volumeMounts: - name: karmada-config mountPath: /etc/karmada/config - - name: cert - mountPath: /var/serving-cert + - name: server-cert + mountPath: /etc/karmada/pki/server readOnly: true volumes: - name: karmada-config secret: secretName: karmada-webhook-config - - name: cert + - name: server-cert secret: - secretName: webhook-cert + secretName: karmada-webhook-cert --- apiVersion: v1 kind: Service diff --git a/artifacts/deploy/kube-controller-manager.yaml b/artifacts/deploy/kube-controller-manager.yaml index 9d77a7806052..1eb16c7bebd5 100644 --- a/artifacts/deploy/kube-controller-manager.yaml +++ b/artifacts/deploy/kube-controller-manager.yaml @@ -33,6 +33,9 @@ spec: topologyKey: kubernetes.io/hostname priorityClassName: system-node-critical containers: + # --client-ca-file verifies the cert of its client like kubelet and other controller + # --cluster-signing-key-file is used for signing certificates + # --root-ca-file is stored in service account type secret - command: - kube-controller-manager - --allocate-node-cidrs=true @@ -40,16 +43,16 @@ spec: - --authentication-kubeconfig=/etc/karmada/config/karmada.config - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --bind-address=0.0.0.0 - - --client-ca-file=/etc/karmada/pki/ca.crt + - --client-ca-file=/etc/karmada/pki/ca/tls.crt - --cluster-cidr=10.244.0.0/16 - --cluster-name=karmada - - --cluster-signing-cert-file=/etc/karmada/pki/ca.crt - - --cluster-signing-key-file=/etc/karmada/pki/ca.key + - --cluster-signing-cert-file=/etc/karmada/pki/ca/tls.crt + - --cluster-signing-key-file=/etc/karmada/pki/ca/tls.key - --controllers=namespace,garbagecollector,serviceaccount-token,ttl-after-finished,bootstrapsigner,tokencleaner,csrapproving,csrcleaner,csrsigning,clusterrole-aggregation - --leader-elect=true - --node-cidr-mask-size=24 - - --root-ca-file=/etc/karmada/pki/ca.crt - - --service-account-private-key-file=/etc/karmada/pki/karmada.key + - --root-ca-file=/etc/karmada/pki/ca/tls.crt + - --service-account-private-key-file=/etc/karmada/pki/service-account-key-pair/sa.key - --service-cluster-ip-range=10.96.0.0/12 - --use-service-account-credentials=true - --v=4 @@ -72,13 +75,19 @@ spec: volumeMounts: - name: karmada-config mountPath: /etc/karmada/config - - mountPath: /etc/karmada/pki - name: karmada-certs + - name: ca-cert + mountPath: /etc/karmada/pki/ca + readOnly: true + - name: service-account-key-pair + mountPath: /etc/karmada/pki/service-account-key-pair readOnly: true volumes: - name: karmada-config secret: secretName: kube-controller-manager-config - - name: karmada-certs + - name: ca-cert + secret: + secretName: kube-controller-manager-ca-cert + - name: service-account-key-pair secret: - secretName: karmada-cert-secret + secretName: kube-controller-manager-service-account-key-pair diff --git a/examples/customresourceinterpreter/karmada-interpreter-webhook-example.yaml b/examples/customresourceinterpreter/karmada-interpreter-webhook-example.yaml index 8a85ac347ae1..66456f30073e 100644 --- a/examples/customresourceinterpreter/karmada-interpreter-webhook-example.yaml +++ b/examples/customresourceinterpreter/karmada-interpreter-webhook-example.yaml @@ -28,7 +28,7 @@ spec: - --kubeconfig=/etc/karmada/config/karmada.config - --bind-address=0.0.0.0 - --secure-port=8445 - - --cert-dir=/var/serving-cert + - --cert-dir=/etc/karmada/pki/server - --v=4 ports: - containerPort: 8445 @@ -40,16 +40,16 @@ spec: volumeMounts: - name: karmada-config mountPath: /etc/karmada/config - - name: cert - mountPath: /var/serving-cert + - name: server-cert + mountPath: /etc/karmada/pki/server readOnly: true volumes: - name: karmada-config secret: secretName: karmada-interpreter-webhook-example-config - - name: cert + - name: server-cert secret: - secretName: webhook-cert + secretName: karmada-interpreter-webhook-example-cert --- apiVersion: v1 kind: Service diff --git a/hack/deploy-karmada.sh b/hack/deploy-karmada.sh index 6ac9a0502fde..c95906378204 100755 --- a/hack/deploy-karmada.sh +++ b/hack/deploy-karmada.sh @@ -86,7 +86,7 @@ fi HOST_CLUSTER_TYPE=${3:-"local"} # the default of host cluster type is local, i.e. cluster created by kind. # generate a secret to store the certificates -function generate_cert_secret { +function generate_cert_related_secrets { local karmada_ca local karmada_ca_key karmada_ca=$(base64 < "${ROOT_CA_FILE}" | tr -d '\r\n') @@ -94,37 +94,36 @@ function generate_cert_secret { local TEMP_PATH TEMP_PATH=$(mktemp -d) - - cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-cert-secret.yaml "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-webhook-cert-secret.yaml "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml - - sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{ca_key}}/${karmada_ca_key}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{client_crt}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{apiserver_crt}}/${KARMADA_APISERVER_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{apiserver_key}}/${KARMADA_APISERVER_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - - sed -i'' -e "s/{{front_proxy_ca_crt}}/${FRONT_PROXY_CA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{front_proxy_client_crt}}/${FRONT_PROXY_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{front_proxy_client_key}}/${FRONT_PROXY_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - - sed -i'' -e "s/{{etcd_ca_crt}}/${ETCD_CA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{etcd_server_crt}}/${ETCD_SERVER_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{etcd_server_key}}/${ETCD_SERVER_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{etcd_client_crt}}/${ETCD_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i'' -e "s/{{etcd_client_key}}/${ETCD_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - - sed -i'' -e "s/{{server_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml - sed -i'' -e "s/{{server_certificate}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml - - kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml - + echo ${TEMP_PATH} + + # 1. generate secret with secret cert + generate_cert_secret karmada-apiserver ${karmada_ca} ${SERVER_CRT} ${SERVER_KEY} + generate_cert_secret karmada-aggregated-apiserver ${karmada_ca} ${SERVER_CRT} ${SERVER_KEY} + generate_cert_secret karmada-metrics-adapter ${karmada_ca} ${SERVER_CRT} ${SERVER_KEY} + generate_cert_secret karmada-search ${karmada_ca} ${SERVER_CRT} ${SERVER_KEY} + generate_cert_secret karmada-webhook ${karmada_ca} ${SERVER_CRT} ${SERVER_KEY} + generate_cert_secret karmada-interpreter-webhook-example ${karmada_ca} ${SERVER_CRT} ${SERVER_KEY} + generate_cert_secret etcd ${karmada_ca} ${ETCD_SERVER_CRT} ${ETCD_SERVER_KEY} + + # 2. generate secret with client cert + generate_cert_secret karmada-apiserver-etcd-client ${karmada_ca} ${ETCD_CLIENT_CRT} ${ETCD_CLIENT_KEY} + generate_cert_secret karmada-apiserver-front-proxy-client ${karmada_ca} ${FRONT_PROXY_CLIENT_CRT} ${FRONT_PROXY_CLIENT_KEY} + generate_cert_secret karmada-aggregated-apiserver-etcd-client ${karmada_ca} ${ETCD_CLIENT_CRT} ${ETCD_CLIENT_KEY} + generate_cert_secret karmada-search-etcd-client ${karmada_ca} ${ETCD_CLIENT_CRT} ${ETCD_CLIENT_KEY} + generate_cert_secret etcd-etcd-client ${karmada_ca} ${ETCD_CLIENT_CRT} ${ETCD_CLIENT_KEY} + generate_cert_secret karmada-scheduler-scheduler-estimator-client ${karmada_ca} ${CLIENT_CRT} ${CLIENT_KEY} + generate_cert_secret karmada-descheduler-scheduler-estimator-client ${karmada_ca} ${CLIENT_CRT} ${CLIENT_KEY} + + # 3. generate secret with ca cert or sa key + generate_ca_cert_secret kube-controller-manager ${karmada_ca} ${karmada_ca_key} + generate_key_pair_secret kube-controller-manager ${SA_PUB} ${SA_KEY} + generate_key_pair_secret karmada-apiserver ${SA_PUB} ${SA_KEY} + + # 4. generate secret with karmada config components=(karmada-aggregated-apiserver karmada-controller-manager kube-controller-manager karmada-scheduler karmada-descheduler karmada-metrics-adapter karmada-search karmada-webhook karmada-interpreter-webhook-example) for component in "${components[@]}" do - generate_config_secret ${component} ${karmada_ca} ${KARMADA_CRT} ${KARMADA_KEY} + generate_config_secret ${component} ${karmada_ca} ${CLIENT_CRT} ${CLIENT_KEY} done rm -rf "${TEMP_PATH}" @@ -137,6 +136,27 @@ function generate_config_secret() { unset component ca_crt client_crt client_key } +function generate_cert_secret() { + export name=$1 ca_crt=$2 tls_crt=$3 tls_key=$4 + envsubst < "${REPO_ROOT}"/artifacts/deploy/karmada-cert-secret.yaml > "${TEMP_PATH}"/${name}-cert-secret.yaml + kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/${name}-cert-secret.yaml + unset name ca_crt tls_crt tls_key +} + +function generate_ca_cert_secret() { + export component=$1 ca_crt=$2 ca_key=$3 + envsubst < "${REPO_ROOT}"/artifacts/deploy/karmada-ca-cert-secret.yaml > "${TEMP_PATH}"/${component}-ca-cert-secret.yaml + kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/${component}-ca-cert-secret.yaml + unset component ca_crt ca_key +} + +function generate_key_pair_secret() { + export component=$1 sa_pub=$2 sa_key=$3 + envsubst < "${REPO_ROOT}"/artifacts/deploy/karmada-key-pair-secret.yaml > "${TEMP_PATH}"/${component}-key-pair-secret.yaml + kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/${component}-key-pair-secret.yaml + unset component sa_pub sa_key +} + # install Karmada's APIs function installCRDs() { local context_name=$1 @@ -157,31 +177,31 @@ util::cmd_must_exist "openssl" util::cmd_must_exist_cfssl ${CFSSL_VERSION} # create CA signers util::create_signing_certkey "" "${CERT_DIR}" ca karmada '"client auth","server auth"' -util::create_signing_certkey "" "${CERT_DIR}" front-proxy-ca front-proxy-ca '"client auth","server auth"' -util::create_signing_certkey "" "${CERT_DIR}" etcd-ca etcd-ca '"client auth","server auth"' # signs a certificate -util::create_certkey "" "${CERT_DIR}" "ca" karmada system:admin "system:masters" kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" "${interpreter_webhook_example_service_external_ip_address}" -util::create_certkey "" "${CERT_DIR}" "ca" apiserver karmada-apiserver "" "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" $(util::get_apiserver_ip_from_kubeconfig "${HOST_CLUSTER_NAME}") -util::create_certkey "" "${CERT_DIR}" "front-proxy-ca" front-proxy-client front-proxy-client "" kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" -util::create_certkey "" "${CERT_DIR}" "etcd-ca" etcd-server etcd-server "" kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" -util::create_certkey "" "${CERT_DIR}" "etcd-ca" etcd-client etcd-client "" "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" +karmadaAltNames=("*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" $(util::get_apiserver_ip_from_kubeconfig "${HOST_CLUSTER_NAME}") "${interpreter_webhook_example_service_external_ip_address}") +util::create_certkey "" "${CERT_DIR}" "ca" server server "" "${karmadaAltNames[@]}" +util::create_certkey "" "${CERT_DIR}" "ca" client system:admin system:masters "${karmadaAltNames[@]}" +util::create_certkey "" "${CERT_DIR}" "ca" front-proxy-client front-proxy-client "" "${karmadaAltNames[@]}" +util::create_certkey "" "${CERT_DIR}" "ca" etcd-server etcd-server "" "${karmadaAltNames[@]}" +util::create_certkey "" "${CERT_DIR}" "ca" etcd-client etcd-client "" "${karmadaAltNames[@]}" +util::create_key_pair "" "${CERT_DIR}" "sa" # create namespace for control plane components kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml" -KARMADA_CRT=$(base64 < "${CERT_DIR}/karmada.crt" | tr -d '\r\n') -KARMADA_KEY=$(base64 < "${CERT_DIR}/karmada.key" | tr -d '\r\n') -KARMADA_APISERVER_CRT=$(base64 < "${CERT_DIR}/apiserver.crt" | tr -d '\r\n') -KARMADA_APISERVER_KEY=$(base64 < "${CERT_DIR}/apiserver.key" | tr -d '\r\n') -FRONT_PROXY_CA_CRT=$(base64 < "${CERT_DIR}/front-proxy-ca.crt" | tr -d '\r\n') +SERVER_CRT=$(base64 < "${CERT_DIR}/server.crt" | tr -d '\r\n') +SERVER_KEY=$(base64 < "${CERT_DIR}/server.key" | tr -d '\r\n') +CLIENT_CRT=$(base64 < "${CERT_DIR}/client.crt" | tr -d '\r\n') +CLIENT_KEY=$(base64 < "${CERT_DIR}/client.key" | tr -d '\r\n') FRONT_PROXY_CLIENT_CRT=$(base64 < "${CERT_DIR}/front-proxy-client.crt" | tr -d '\r\n') FRONT_PROXY_CLIENT_KEY=$(base64 < "${CERT_DIR}/front-proxy-client.key" | tr -d '\r\n') -ETCD_CA_CRT=$(base64 < "${CERT_DIR}/etcd-ca.crt" | tr -d '\r\n') ETCD_SERVER_CRT=$(base64 < "${CERT_DIR}/etcd-server.crt" | tr -d '\r\n') ETCD_SERVER_KEY=$(base64 < "${CERT_DIR}/etcd-server.key" | tr -d '\r\n') ETCD_CLIENT_CRT=$(base64 < "${CERT_DIR}/etcd-client.crt" | tr -d '\r\n') ETCD_CLIENT_KEY=$(base64 < "${CERT_DIR}/etcd-client.key" | tr -d '\r\n') -generate_cert_secret +SA_PUB=$(base64 < "${CERT_DIR}/sa.pub" | tr -d '\r\n') +SA_KEY=$(base64 < "${CERT_DIR}/sa.key" | tr -d '\r\n') +generate_cert_related_secrets # deploy karmada etcd kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-etcd.yaml" @@ -245,7 +265,7 @@ else fi # write karmada api server config to kubeconfig file -util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/karmada.crt" "${CERT_DIR}/karmada.key" "${KARMADA_APISERVER_IP}" "${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver +util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/client.crt" "${CERT_DIR}/client.key" "${KARMADA_APISERVER_IP}" "${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver # deploy kube controller manager cp "${REPO_ROOT}"/artifacts/deploy/kube-controller-manager.yaml "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml diff --git a/hack/util.sh b/hack/util.sh index ebc1ecd931d4..021fa242c272 100755 --- a/hack/util.sh +++ b/hack/util.sh @@ -243,6 +243,18 @@ function util::create_certkey { EOF } +# util::create_key_pair generates a new public and private key pair. +function util::create_key_pair { + local sudo=$1 + local dest_dir=$2 + local name=$3 + ${sudo} /usr/bin/env bash -e <