diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl index 29c1fcd6ab6..ec455a8869f 100644 --- a/charts/tidb-cluster/templates/_helpers.tpl +++ b/charts/tidb-cluster/templates/_helpers.tpl @@ -37,6 +37,13 @@ config-file: |- {{- if .Values.pd.config }} {{ .Values.pd.config | indent 2 }} {{- end -}} + {{- if .Values.enableTLSCluster }} + [security] + cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cert-path = "/var/lib/pd-tls/pd.crt" + key-path = "/var/lib/pd-tls/pd.key" + {{- end -}} + {{- end -}} {{- define "pd-configmap.data-digest" -}} @@ -53,6 +60,13 @@ config-file: |- {{- if .Values.tikv.config }} {{ .Values.tikv.config | indent 2 }} {{- end -}} + {{- if .Values.enableTLSCluster }} + [security] + ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cert-path = "/var/lib/tikv-tls/tikv.crt" + key-path = "/var/lib/tikv-tls/tikv.key" + {{- end -}} + {{- end -}} {{- define "tikv-configmap.data-digest" -}} @@ -73,6 +87,20 @@ config-file: |- {{- if .Values.tidb.config }} {{ .Values.tidb.config | indent 2 }} {{- end -}} + {{- if or .Values.enableTLSCluster .Values.enableTLSClient }} + [security] + {{- end -}} + {{- if .Values.enableTLSCluster }} + cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cluster-ssl-cert = "/var/lib/tidb-tls/tidb.crt" + cluster-ssl-key = "/var/lib/tidb-tls/tidb.key" + {{- end -}} + {{- if .Values.tidb.enableTLSClient }} + ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + ssl-cert = "/var/lib/tidb-tls/tidb.crt" + ssl-key = "/var/lib/tidb-tls/tidb.key" + {{- end -}} + {{- end -}} {{- define "tidb-configmap.data-digest" -}} diff --git a/charts/tidb-cluster/templates/discovery-deployment.yaml b/charts/tidb-cluster/templates/discovery-deployment.yaml index a661efacb41..137d0a9abcb 100644 --- a/charts/tidb-cluster/templates/discovery-deployment.yaml +++ b/charts/tidb-cluster/templates/discovery-deployment.yaml @@ -45,4 +45,14 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - +{{- if .Values.enableTLSCluster }} + volumeMounts: + - mountPath: /var/lib/tls + name: tls + readOnly: true + volumes: + - name: tls + secret: + defaultMode: 420 + secretName: client-tls +{{- end -}} diff --git a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl index cca58378a69..7c1747d7dcf 100644 --- a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl @@ -58,12 +58,14 @@ while true; do fi done +SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }} + ARGS="--data-dir=/var/lib/pd \ --name=${POD_NAME} \ ---peer-urls=http://0.0.0.0:2380 \ ---advertise-peer-urls=http://${domain}:2380 \ ---client-urls=http://0.0.0.0:2379 \ ---advertise-client-urls=http://${domain}:2379 \ +--peer-urls=${SCHEME}://0.0.0.0:2380 \ +--advertise-peer-urls=${SCHEME}://${domain}:2380 \ +--client-urls=${SCHEME}://0.0.0.0:2379 \ +--advertise-client-urls=${SCHEME}://${domain}:2379 \ --config=/etc/pd/pd.toml \ " diff --git a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl index fabb17b6066..5c598d63d56 100644 --- a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl @@ -28,9 +28,11 @@ then tail -f /dev/null fi +SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }} + # Use HOSTNAME if POD_NAME is unset for backward compatibility. POD_NAME=${POD_NAME:-$HOSTNAME} -ARGS="--pd=${CLUSTER_NAME}-pd:2379 \ +ARGS="--pd=${SCHEME}://${CLUSTER_NAME}-pd:2379 \ --advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20160 \ --addr=0.0.0.0:20160 \ --status-addr=0.0.0.0:20180 \ diff --git a/charts/tidb-cluster/templates/tidb-cluster.yaml b/charts/tidb-cluster/templates/tidb-cluster.yaml index 0f86e6c0064..39d6fb3ab02 100644 --- a/charts/tidb-cluster/templates/tidb-cluster.yaml +++ b/charts/tidb-cluster/templates/tidb-cluster.yaml @@ -20,6 +20,8 @@ metadata: spec: pvReclaimPolicy: {{ .Values.pvReclaimPolicy }} timezone: {{ .Values.timezone | default "UTC" }} + enableTLSCluster: {{ .Values.enableTLSCluster | default false }} + enableTLSClient: {{ .Values.enableTLSClient | default false }} services: {{ toYaml .Values.services | indent 4 }} schedulerName: {{ .Values.schedulerName | default "default-scheduler" }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 8bd863d4ec4..120d2730dba 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -47,6 +47,11 @@ discovery: # if the ConfigMap was not changed. enableConfigMapRollout: true +# Whether enable TLS connections between server nodes. +# When enabled, PD/TiDB/TiKV will use TLS encrypted connections to transfer data between each node, +# certificates will be generated automatically (if not already present). +enableTLSCluster: false + pd: # Please refer to https://github.com/pingcap/pd/blob/master/conf/config.toml for the default # pd configurations (change to the tags of your pd version), @@ -247,6 +252,7 @@ tidb: config: | [log] level = "info" + # # Here are some parameters you MUST customize (Please configure in the above 'tidb.config' section): # [performance] # # Normally it should be tuned to `tidb.resources.limits.cpu`, for example: 16000m -> 16 @@ -330,6 +336,12 @@ tidb: # the start argument to specify the plugin id (name "-" version) that needs to be loaded, e.g. 'conn_limit-1'. list: ["whitelist-1"] + # Whether enable TLS connection between TiDB server and MySQL client. + # When enabled, TiDB will accept TLS encrypted connections from MySQL client, certificates will be generated + # automatically. + # Note: TLS connection is not forced on the server side, plain connections are also accepted after enableing. + enableTLSClient: false + # mysqlClient is used to set password for TiDB # it must has Python MySQL client installed mysqlClient: diff --git a/charts/tidb-operator/templates/controller-manager-deployment.yaml b/charts/tidb-operator/templates/controller-manager-deployment.yaml index ef08fd37f9b..c744340ff9a 100644 --- a/charts/tidb-operator/templates/controller-manager-deployment.yaml +++ b/charts/tidb-operator/templates/controller-manager-deployment.yaml @@ -56,6 +56,15 @@ spec: fieldPath: metadata.namespace - name: TZ value: {{ .Values.timezone | default "UTC" }} + volumeMounts: + - mountPath: /var/lib/tls + name: tls + readOnly: true + volumes: + - name: tls + secret: + defaultMode: 420 + secretName: client-tls {{- with .Values.controllerManager.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} @@ -67,4 +76,4 @@ spec: {{- with .Values.controllerManager.tolerations }} tolerations: {{ toYaml . | indent 8 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go b/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go index 47c45b02793..8d999cced30 100644 --- a/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go +++ b/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go @@ -158,3 +158,10 @@ func (tc *TidbCluster) TiKVIsAvailable() bool { func (tc *TidbCluster) GetClusterID() string { return tc.Status.ClusterID } + +func (tc *TidbCluster) Scheme() string { + if tc.Spec.EnableTLSCluster { + return "https" + } + return "http" +} diff --git a/pkg/apis/pingcap.com/v1alpha1/types.go b/pkg/apis/pingcap.com/v1alpha1/types.go index 718abe19cdb..ec0a6495777 100644 --- a/pkg/apis/pingcap.com/v1alpha1/types.go +++ b/pkg/apis/pingcap.com/v1alpha1/types.go @@ -93,6 +93,8 @@ type TidbClusterSpec struct { Services []Service `json:"services,omitempty"` PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"` Timezone string `json:"timezone,omitempty"` + // Enable TLS connection between TiDB server compoments + EnableTLSCluster bool `json:"enableTLSCluster,omitempty"` } // TidbClusterStatus represents the current status of a tidb cluster. @@ -103,7 +105,7 @@ type TidbClusterStatus struct { TiDB TiDBStatus `json:"tidb,omitempty"` } -// PDSpec contains details of PD member +// PDSpec contains details of PD members type PDSpec struct { ContainerSpec Replicas int32 `json:"replicas"` @@ -115,7 +117,7 @@ type PDSpec struct { StorageClassName string `json:"storageClassName,omitempty"` } -// TiDBSpec contains details of PD member +// TiDBSpec contains details of TiDB members type TiDBSpec struct { ContainerSpec Replicas int32 `json:"replicas"` @@ -129,6 +131,7 @@ type TiDBSpec struct { MaxFailoverCount int32 `json:"maxFailoverCount,omitempty"` SeparateSlowLog bool `json:"separateSlowLog,omitempty"` SlowLogTailer TiDBSlowLogTailerSpec `json:"slowLogTailer,omitempty"` + EnableTLSClient bool `json:"enableTLSClient,omitempty"` } // TiDBSlowLogTailerSpec represents an optional log tailer sidecar with TiDB @@ -136,7 +139,7 @@ type TiDBSlowLogTailerSpec struct { ContainerSpec } -// TiKVSpec contains details of PD member +// TiKVSpec contains details of TiKV members type TiKVSpec struct { ContainerSpec Replicas int32 `json:"replicas"` diff --git a/pkg/controller/pd_control.go b/pkg/controller/pd_control.go index 597f1fa4089..715eb3d03ec 100644 --- a/pkg/controller/pd_control.go +++ b/pkg/controller/pd_control.go @@ -20,7 +20,7 @@ import ( // GetPDClient gets the pd client from the TidbCluster func GetPDClient(pdControl pdapi.PDControlInterface, tc *v1alpha1.TidbCluster) pdapi.PDClient { - return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()) + return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster) } // NewFakePDClient creates a fake pdclient that is set as the pd client diff --git a/pkg/controller/pod_control.go b/pkg/controller/pod_control.go index 824a1da0a2f..e80f95105a2 100644 --- a/pkg/controller/pod_control.go +++ b/pkg/controller/pod_control.go @@ -111,7 +111,8 @@ func (rpc *realPodControl) UpdateMetaInfo(tc *v1alpha1.TidbCluster, pod *corev1. clusterID := labels[label.ClusterIDLabelKey] memberID := labels[label.MemberIDLabelKey] storeID := labels[label.StoreIDLabelKey] - pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName) + + pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName, tc.Spec.EnableTLSCluster) if labels[label.ClusterIDLabelKey] == "" { cluster, err := pdClient.GetCluster() if err != nil { diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index 5cd376b6890..f9e150b787f 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -14,6 +14,7 @@ package controller import ( + "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -55,18 +56,37 @@ type defaultTiDBControl struct { // NewDefaultTiDBControl returns a defaultTiDBControl instance func NewDefaultTiDBControl() TiDBControlInterface { - httpClient := &http.Client{Timeout: timeout} - return &defaultTiDBControl{httpClient: httpClient} + return &defaultTiDBControl{httpClient: &http.Client{Timeout: timeout}} +} + +func (tdc *defaultTiDBControl) useTLSHTTPClient(enableTLS bool) error { + if enableTLS { + rootCAs, err := httputil.ReadCACerts() + if err != nil { + return err + } + config := &tls.Config{ + RootCAs: rootCAs, + } + tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} + } + return nil } func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bool { tcName := tc.GetName() ns := tc.GetNamespace() + scheme := tc.Scheme() result := map[string]bool{} + + if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil { + return result + } + for i := 0; i < int(tc.TiDBRealReplicas()); i++ { hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), i) - url := fmt.Sprintf("http://%s.%s.%s:10080/status", hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/status", scheme, hostName, TiDBPeerMemberName(tcName), ns) _, err := tdc.getBodyOK(url) if err != nil { result[hostName] = false @@ -80,9 +100,13 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) { tcName := tc.GetName() ns := tc.GetNamespace() + scheme := tc.Scheme() + if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil { + return false, err + } hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) - url := fmt.Sprintf("http://%s.%s.%s:10080/ddl/owner/resign", hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/ddl/owner/resign", scheme, hostName, TiDBPeerMemberName(tcName), ns) req, err := http.NewRequest("POST", url, nil) if err != nil { return false, err @@ -105,9 +129,13 @@ func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) (*dbInfo, error) { tcName := tc.GetName() ns := tc.GetNamespace() + scheme := tc.Scheme() + if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil { + return nil, err + } hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) - url := fmt.Sprintf("http://%s.%s.%s:10080/info", hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/info", scheme, hostName, TiDBPeerMemberName(tcName), ns) req, err := http.NewRequest("POST", url, nil) if err != nil { return nil, err @@ -136,9 +164,13 @@ func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) func (tdc *defaultTiDBControl) GetSettings(tc *v1alpha1.TidbCluster, ordinal int32) (*config.Config, error) { tcName := tc.GetName() ns := tc.GetNamespace() + scheme := tc.Scheme() + if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil { + return nil, err + } hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) - url := fmt.Sprintf("http://%s.%s.%s:10080/settings", hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/settings", scheme, hostName, TiDBPeerMemberName(tcName), ns) req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 7a8f4328181..f4e45df934d 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -94,10 +94,10 @@ func (td *tidbDiscovery) Discover(advertisePeerUrl string) (string, error) { if len(currentCluster.peers) == int(replicas) { delete(currentCluster.peers, podName) - return fmt.Sprintf("--initial-cluster=%s=http://%s", podName, advertisePeerUrl), nil + return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, tc.Scheme(), advertisePeerUrl), nil } - pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()) + pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster) membersInfo, err := pdClient.GetMembers() if err != nil { return "", err diff --git a/pkg/httputil/httputil.go b/pkg/httputil/httputil.go index 84b0c70780c..91bf94adc83 100644 --- a/pkg/httputil/httputil.go +++ b/pkg/httputil/httputil.go @@ -1,6 +1,8 @@ package httputil import ( + "crypto/tls" + "crypto/x509" "fmt" "io" "io/ioutil" @@ -9,6 +11,12 @@ import ( "github.com/golang/glog" ) +const ( + k8sCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + clientCert = "/var/lib/tls/client.crt" + clientKey = "/var/lib/tls/client.key" +) + // DeferClose captures and prints the error from closing (if an error occurs). // This is designed to be used in a defer statement. func DeferClose(c io.Closer) { @@ -44,3 +52,35 @@ func GetBodyOK(httpClient *http.Client, apiURL string) ([]byte, error) { } return body, err } + +func ReadCACerts() (*x509.CertPool, error) { + // try to load system CA certs + rootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + + // load k8s CA cert + caCert, err := ioutil.ReadFile(k8sCAFile) + if err != nil { + return nil, fmt.Errorf("fail to read CA file %s, error: %v", k8sCAFile, err) + } + if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { + glog.Warningf("fail to append CA file to pool, using system CAs only") + } + return rootCAs, nil +} + +func ReadCerts() (*x509.CertPool, tls.Certificate, error) { + rootCAs, err := ReadCACerts() + if err != nil { + return rootCAs, tls.Certificate{}, err + } + + // load client cert + cert, err := tls.LoadX509KeyPair(clientCert, clientKey) + return rootCAs, cert, err +} diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index 222edc7cb45..359c8b51b73 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -437,6 +437,12 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"}, {Name: v1alpha1.PDMemberType.String(), MountPath: "/var/lib/pd"}, } + if tc.Spec.EnableTLSCluster { + volMounts = append(volMounts, corev1.VolumeMount{ + Name: "pd-tls", ReadOnly: true, MountPath: "/var/lib/pd-tls", + }) + } + vols := []corev1.Volume{ annVolume, {Name: "config", @@ -460,6 +466,15 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) }, }, } + if tc.Spec.EnableTLSCluster { + vols = append(vols, corev1.Volume{ + Name: "pd-tls", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: controller.PDMemberName(tcName), + }, + }, + }) + } var q resource.Quantity var err error diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 25686f55d8a..c9fce8b81a4 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -229,6 +229,12 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust {Name: "config", ReadOnly: true, MountPath: "/etc/tidb"}, {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"}, } + if tc.Spec.EnableTLSCluster { + volMounts = append(volMounts, corev1.VolumeMount{ + Name: "tidb-tls", ReadOnly: true, MountPath: "/var/lib/tidb-tls", + }) + } + vols := []corev1.Volume{ annVolume, {Name: "config", VolumeSource: corev1.VolumeSource{ @@ -248,6 +254,15 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust }}, }, } + if tc.Spec.EnableTLSCluster { + vols = append(vols, corev1.Volume{ + Name: "tidb-tls", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: controller.TiDBMemberName(tcName), + }, + }, + }) + } var containers []corev1.Container if tc.Spec.TiDB.SeparateSlowLog { @@ -298,6 +313,10 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust }, } + scheme := corev1.URISchemeHTTP + if tc.Spec.EnableTLSCluster { + scheme = corev1.URISchemeHTTPS + } containers = append(containers, corev1.Container{ Name: v1alpha1.TiDBMemberType.String(), Image: tc.Spec.TiDB.Image, @@ -321,8 +340,9 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust ReadinessProbe: &corev1.Probe{ Handler: corev1.Handler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/status", - Port: intstr.FromInt(10080), + Path: "/status", + Port: intstr.FromInt(10080), + Scheme: scheme, }, }, InitialDelaySeconds: int32(10), diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 4e36f05f7bc..9293e924ae1 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -279,6 +279,12 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) {Name: "config", ReadOnly: true, MountPath: "/etc/tikv"}, {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"}, } + if tc.Spec.EnableTLSCluster { + volMounts = append(volMounts, corev1.VolumeMount{ + Name: "tikv-tls", ReadOnly: true, MountPath: "/var/lib/tikv-tls", + }) + } + vols := []corev1.Volume{ annVolume, {Name: "config", VolumeSource: corev1.VolumeSource{ @@ -298,6 +304,15 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) }}, }, } + if tc.Spec.EnableTLSCluster { + vols = append(vols, corev1.Volume{ + Name: "tikv-tls", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: controller.TiKVMemberName(tcName), + }, + }, + }) + } var q resource.Quantity var err error diff --git a/pkg/manager/member/tikv_upgrader.go b/pkg/manager/member/tikv_upgrader.go index a7d6aa41091..f4bf8113a06 100644 --- a/pkg/manager/member/tikv_upgrader.go +++ b/pkg/manager/member/tikv_upgrader.go @@ -200,7 +200,7 @@ func (tku *tikvUpgrader) endEvictLeader(tc *v1alpha1.TidbCluster, ordinal int32) return err } - err = tku.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()).EndEvictLeader(storeID) + err = tku.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster).EndEvictLeader(storeID) if err != nil { glog.Errorf("tikv upgrader: failed to end evict leader storeID: %d ordinal: %d, %v", storeID, ordinal, err) return err diff --git a/pkg/pdapi/pdapi.go b/pkg/pdapi/pdapi.go index f96ff84200e..b60e51379cc 100644 --- a/pkg/pdapi/pdapi.go +++ b/pkg/pdapi/pdapi.go @@ -15,6 +15,7 @@ package pdapi import ( "bytes" + "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -42,7 +43,7 @@ type Namespace string // PDControlInterface is an interface that knows how to manage and get tidb cluster's PD client type PDControlInterface interface { // GetPDClient provides PDClient of the tidb cluster. - GetPDClient(Namespace, string) PDClient + GetPDClient(Namespace, string, bool) PDClient } // defaultPDControl is the default implementation of PDControlInterface. @@ -57,24 +58,29 @@ func NewDefaultPDControl() PDControlInterface { } // GetPDClient provides a PDClient of real pd cluster,if the PDClient not existing, it will create new one. -func (pdc *defaultPDControl) GetPDClient(namespace Namespace, tcName string) PDClient { +func (pdc *defaultPDControl) GetPDClient(namespace Namespace, tcName string, tlsEnabled bool) PDClient { pdc.mutex.Lock() defer pdc.mutex.Unlock() - key := pdClientKey(namespace, tcName) + + scheme := "http" + if tlsEnabled { + scheme = "https" + } + key := pdClientKey(scheme, namespace, tcName) if _, ok := pdc.pdClients[key]; !ok { - pdc.pdClients[key] = NewPDClient(PdClientURL(namespace, tcName), timeout) + pdc.pdClients[key] = NewPDClient(PdClientURL(namespace, tcName, scheme), timeout, tlsEnabled) } return pdc.pdClients[key] } // pdClientKey returns the pd client key -func pdClientKey(namespace Namespace, clusterName string) string { - return fmt.Sprintf("%s.%s", clusterName, string(namespace)) +func pdClientKey(scheme string, namespace Namespace, clusterName string) string { + return fmt.Sprintf("%s.%s.%s", scheme, clusterName, string(namespace)) } // pdClientUrl builds the url of pd client -func PdClientURL(namespace Namespace, clusterName string) string { - return fmt.Sprintf("http://%s-pd.%s:2379", clusterName, string(namespace)) +func PdClientURL(namespace Namespace, clusterName string, scheme string) string { + return fmt.Sprintf("%s://%s-pd.%s:2379", scheme, clusterName, string(namespace)) } // PDClient provides pd server's api @@ -134,10 +140,26 @@ type pdClient struct { } // NewPDClient returns a new PDClient -func NewPDClient(url string, timeout time.Duration) PDClient { +func NewPDClient(url string, timeout time.Duration, tlsEnabled bool) PDClient { + httpClient := &http.Client{Timeout: timeout} + if tlsEnabled { + rootCAs, cert, err := httputil.ReadCerts() + if err != nil { + glog.Errorf("fail to load certs, fallback to plain connection, err: %s", err) + } else { + config := &tls.Config{ + RootCAs: rootCAs, + Certificates: []tls.Certificate{cert}, + } + httpClient = &http.Client{ + Timeout: timeout, + Transport: &http.Transport{TLSClientConfig: config}, + } + } + } return &pdClient{ url: url, - httpClient: &http.Client{Timeout: timeout}, + httpClient: httpClient, } } @@ -592,7 +614,7 @@ func NewFakePDControl() *FakePDControl { } func (fpc *FakePDControl) SetPDClient(namespace Namespace, tcName string, pdclient PDClient) { - fpc.defaultPDControl.pdClients[pdClientKey(namespace, tcName)] = pdclient + fpc.defaultPDControl.pdClients[pdClientKey("http", namespace, tcName)] = pdclient } type ActionType string diff --git a/pkg/pdapi/pdapi_test.go b/pkg/pdapi/pdapi_test.go index 96a4157c927..c6f949a2885 100644 --- a/pkg/pdapi/pdapi_test.go +++ b/pkg/pdapi/pdapi_test.go @@ -73,7 +73,7 @@ func TestHealth(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetHealth() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(&HealthInfo{healths})) @@ -114,7 +114,7 @@ func TestGetConfig(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetConfig() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(config)) @@ -152,7 +152,7 @@ func TestGetCluster(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetCluster() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(cluster)) @@ -204,7 +204,7 @@ func TestGetMembers(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetMembers() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(members)) @@ -256,7 +256,7 @@ func TestGetStores(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetStores() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(stores)) @@ -301,7 +301,7 @@ func TestGetStore(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetStore(tc.id) g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(store)) @@ -350,7 +350,7 @@ func TestSetStoreLabels(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, _ := pdClient.SetStoreLabels(id, labels) g.Expect(result).To(Equal(tc.want)) } @@ -440,7 +440,7 @@ func TestDeleteMember(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) err := pdClient.DeleteMember(name) if tc.want { g.Expect(err).NotTo(HaveOccurred(), "check result") @@ -534,7 +534,7 @@ func TestDeleteMemberByID(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) err := pdClient.DeleteMemberByID(id) if tc.want { g.Expect(err).NotTo(HaveOccurred(), "check result") @@ -626,7 +626,7 @@ func TestDeleteStore(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) err := pdClient.DeleteStore(storeID) if tc.want { g.Expect(err).NotTo(HaveOccurred(), "check result") diff --git a/tests/actions.go b/tests/actions.go index 14720bb2422..3110c1743b0 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -994,7 +994,7 @@ func (oa *operatorActions) CheckUpgrade(ctx context.Context, info *TidbClusterCo glog.Errorf("failed to get tidbcluster: %s/%s, %v", ns, tcName, err) continue } - pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()) + pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster) replicas := tc.TiKVRealReplicas() for i := replicas - 1; i >= 0; i-- { diff --git a/tests/failover.go b/tests/failover.go index 6dcbb89d09a..e1d0be82a62 100644 --- a/tests/failover.go +++ b/tests/failover.go @@ -38,7 +38,7 @@ func (oa *operatorActions) TruncateSSTFileThenCheckFailover(info *TidbClusterCon } // checkout pd config - pdCfg, err := oa.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()).GetConfig() + pdCfg, err := oa.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster).GetConfig() if err != nil { glog.Errorf("failed to get the pd config: tc=%s err=%s", info.ClusterName, err.Error()) return err diff --git a/tests/pkg/webhook/pods.go b/tests/pkg/webhook/pods.go index 7b76692de98..be6dc20e0ea 100644 --- a/tests/pkg/webhook/pods.go +++ b/tests/pkg/webhook/pods.go @@ -53,7 +53,7 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { return &reviewResponse } - pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()) + pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster) tidbController := controller.NewDefaultTiDBControl() // if pod is already deleting, return Allowed