Add basic TLS support for TiDB cluster (#750)

* discovery: add HTTPS URL support

* tls: add basic support of certis

* tls: support tls client for pd

* tls: support tls client for controller

* tls: update startup script templates

* tls: fix configs

* tls: fix health check for tidb

* tls: create new pd client when scheme changed

* tls: fix return value when loading CAs

* tls: fix test errors

* tls: fix typo in scripts

* fix test errors

* tls: change config name and use better layout

* tls: refine scheme detection

* tls: fix keys in templates

* tls: fix test cases

* tls: update coding styles

charts/tidb-cluster/templates/_helpers.tpl

@@ -37,6 +37,13 @@ config-file: |-
     {{- if .Values.pd.config }}
 {{ .Values.pd.config | indent 2 }}
     {{- end -}}
+    {{- if .Values.enableTLSCluster }}
+  [security]
+  cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  cert-path = "/var/lib/pd-tls/pd.crt"
+  key-path = "/var/lib/pd-tls/pd.key"
+    {{- end -}}
+
 {{- end -}}
 
 {{- define "pd-configmap.data-digest" -}}
@@ -53,6 +60,13 @@ config-file: |-
     {{- if .Values.tikv.config }}
 {{ .Values.tikv.config | indent 2 }}
     {{- end -}}
+    {{- if .Values.enableTLSCluster }}
+  [security]
+  ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  cert-path = "/var/lib/tikv-tls/tikv.crt"
+  key-path = "/var/lib/tikv-tls/tikv.key"
+    {{- end -}}
+
 {{- end -}}
 
 {{- define "tikv-configmap.data-digest" -}}
@@ -73,6 +87,20 @@ config-file: |-
     {{- if .Values.tidb.config }}
 {{ .Values.tidb.config | indent 2 }}
     {{- end -}}
+    {{- if or .Values.enableTLSCluster .Values.enableTLSClient }}
+  [security]
+    {{- end -}}
+    {{- if .Values.enableTLSCluster }}
+  cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  cluster-ssl-cert = "/var/lib/tidb-tls/tidb.crt"
+  cluster-ssl-key = "/var/lib/tidb-tls/tidb.key"
+    {{- end -}}
+    {{- if .Values.tidb.enableTLSClient }}
+  ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  ssl-cert = "/var/lib/tidb-tls/tidb.crt"
+  ssl-key = "/var/lib/tidb-tls/tidb.key"
+    {{- end -}}
+
 {{- end -}}
 
 {{- define "tidb-configmap.data-digest" -}}

charts/tidb-cluster/templates/discovery-deployment.yaml

@@ -45,4 +45,14 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
-
+{{- if .Values.enableTLSCluster }}
+        volumeMounts:
+        - mountPath: /var/lib/tls
+          name: tls
+          readOnly: true
+      volumes:
+      - name: tls
+        secret:
+          defaultMode: 420
+          secretName: client-tls
+{{- end -}}

charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl

@@ -58,12 +58,14 @@ while true; do
     fi
 done
 
+SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }}
+
 ARGS="--data-dir=/var/lib/pd \
 --name=${POD_NAME} \
---peer-urls=http://0.0.0.0:2380 \
---advertise-peer-urls=http://${domain}:2380 \
---client-urls=http://0.0.0.0:2379 \
---advertise-client-urls=http://${domain}:2379 \
+--peer-urls=${SCHEME}://0.0.0.0:2380 \
+--advertise-peer-urls=${SCHEME}://${domain}:2380 \
+--client-urls=${SCHEME}://0.0.0.0:2379 \
+--advertise-client-urls=${SCHEME}://${domain}:2379 \
 --config=/etc/pd/pd.toml \
 "
 

charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl

@@ -28,9 +28,11 @@ then
 	tail -f /dev/null
 fi
 
+SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }}
+
 # Use HOSTNAME if POD_NAME is unset for backward compatibility.
 POD_NAME=${POD_NAME:-$HOSTNAME}
-ARGS="--pd=${CLUSTER_NAME}-pd:2379 \
+ARGS="--pd=${SCHEME}://${CLUSTER_NAME}-pd:2379 \
 --advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20160 \
 --addr=0.0.0.0:20160 \
 --status-addr=0.0.0.0:20180 \

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -20,6 +20,8 @@ metadata:
 spec:
   pvReclaimPolicy: {{ .Values.pvReclaimPolicy }}
   timezone: {{ .Values.timezone | default "UTC" }}
+  enableTLSCluster: {{ .Values.enableTLSCluster | default false }}
+  enableTLSClient: {{ .Values.enableTLSClient | default false }}
   services:
 {{ toYaml .Values.services | indent 4 }}
   schedulerName: {{ .Values.schedulerName | default "default-scheduler" }}

charts/tidb-cluster/values.yaml

@@ -47,6 +47,11 @@ discovery:
 # if the ConfigMap was not changed.
 enableConfigMapRollout: true
 
+# Whether enable TLS connections between server nodes.
+# When enabled, PD/TiDB/TiKV will use TLS encrypted connections to transfer data between each node,
+# certificates will be generated automatically (if not already present).
+enableTLSCluster: false
+
 pd:
   # Please refer to https://github.com/pingcap/pd/blob/master/conf/config.toml for the default
   # pd configurations (change to the tags of your pd version),
@@ -247,6 +252,7 @@ tidb:
   config: |
     [log]
     level = "info"
+
   # # Here are some parameters you MUST customize (Please configure in the above 'tidb.config' section):
   # [performance]
   #   # Normally it should be tuned to `tidb.resources.limits.cpu`, for example: 16000m -> 16
@@ -330,6 +336,12 @@ tidb:
     # the start argument to specify the plugin id (name "-" version) that needs to be loaded, e.g. 'conn_limit-1'.
     list: ["whitelist-1"]
 
+  # Whether enable TLS connection between TiDB server and MySQL client.
+  # When enabled, TiDB will accept TLS encrypted connections from MySQL client, certificates will be generated
+  # automatically.
+  # Note: TLS connection is not forced on the server side, plain connections are also accepted after enableing.
+  enableTLSClient: false
+
 # mysqlClient is used to set password for TiDB
 # it must has Python MySQL client installed
 mysqlClient:

charts/tidb-operator/templates/controller-manager-deployment.yaml

@@ -56,6 +56,15 @@ spec:
                 fieldPath: metadata.namespace
           - name: TZ
             value: {{ .Values.timezone | default "UTC" }}
+        volumeMounts:
+        - mountPath: /var/lib/tls
+          name: tls
+          readOnly: true
+      volumes:
+      - name: tls
+        secret:
+          defaultMode: 420
+          secretName: client-tls
     {{- with .Values.controllerManager.nodeSelector }}
       nodeSelector:
 {{ toYaml . | indent 8 }}
@@ -67,4 +76,4 @@ spec:
     {{- with .Values.controllerManager.tolerations }}
       tolerations:
 {{ toYaml . | indent 8 }}
-    {{- end }}
+    {{- end }}

pkg/apis/pingcap.com/v1alpha1/tidbcluster.go

@@ -158,3 +158,10 @@ func (tc *TidbCluster) TiKVIsAvailable() bool {
 func (tc *TidbCluster) GetClusterID() string {
 	return tc.Status.ClusterID
 }
+
+func (tc *TidbCluster) Scheme() string {
+	if tc.Spec.EnableTLSCluster {
+		return "https"
+	}
+	return "http"
+}

pkg/apis/pingcap.com/v1alpha1/types.go

@@ -93,6 +93,8 @@ type TidbClusterSpec struct {
 	Services        []Service                            `json:"services,omitempty"`
 	PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"`
 	Timezone        string                               `json:"timezone,omitempty"`
+	// Enable TLS connection between TiDB server compoments
+	EnableTLSCluster bool `json:"enableTLSCluster,omitempty"`
 }
 
 // TidbClusterStatus represents the current status of a tidb cluster.
@@ -103,7 +105,7 @@ type TidbClusterStatus struct {
 	TiDB      TiDBStatus `json:"tidb,omitempty"`
 }
 
-// PDSpec contains details of PD member
+// PDSpec contains details of PD members
 type PDSpec struct {
 	ContainerSpec
 	Replicas         int32               `json:"replicas"`
@@ -115,7 +117,7 @@ type PDSpec struct {
 	StorageClassName string              `json:"storageClassName,omitempty"`
 }
 
-// TiDBSpec contains details of PD member
+// TiDBSpec contains details of TiDB members
 type TiDBSpec struct {
 	ContainerSpec
 	Replicas         int32                 `json:"replicas"`
@@ -129,14 +131,15 @@ type TiDBSpec struct {
 	MaxFailoverCount int32                 `json:"maxFailoverCount,omitempty"`
 	SeparateSlowLog  bool                  `json:"separateSlowLog,omitempty"`
 	SlowLogTailer    TiDBSlowLogTailerSpec `json:"slowLogTailer,omitempty"`
+	EnableTLSClient  bool                  `json:"enableTLSClient,omitempty"`
 }
 
 // TiDBSlowLogTailerSpec represents an optional log tailer sidecar with TiDB
 type TiDBSlowLogTailerSpec struct {
 	ContainerSpec
 }
 
-// TiKVSpec contains details of PD member
+// TiKVSpec contains details of TiKV members
 type TiKVSpec struct {
 	ContainerSpec
 	Replicas         int32               `json:"replicas"`

pkg/controller/pd_control.go

@@ -20,7 +20,7 @@ import (
 
 // GetPDClient gets the pd client from the TidbCluster
 func GetPDClient(pdControl pdapi.PDControlInterface, tc *v1alpha1.TidbCluster) pdapi.PDClient {
-	return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName())
+	return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster)
 }
 
 // NewFakePDClient creates a fake pdclient that is set as the pd client

pkg/controller/pod_control.go

@@ -111,7 +111,8 @@ func (rpc *realPodControl) UpdateMetaInfo(tc *v1alpha1.TidbCluster, pod *corev1.
 	clusterID := labels[label.ClusterIDLabelKey]
 	memberID := labels[label.MemberIDLabelKey]
 	storeID := labels[label.StoreIDLabelKey]
-	pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName)
+
+	pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName, tc.Spec.EnableTLSCluster)
 	if labels[label.ClusterIDLabelKey] == "" {
 		cluster, err := pdClient.GetCluster()
 		if err != nil {

pkg/controller/tidb_control.go

@@ -14,6 +14,7 @@
 package controller
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -55,18 +56,37 @@ type defaultTiDBControl struct {
 
 // NewDefaultTiDBControl returns a defaultTiDBControl instance
 func NewDefaultTiDBControl() TiDBControlInterface {
-	httpClient := &http.Client{Timeout: timeout}
-	return &defaultTiDBControl{httpClient: httpClient}
+	return &defaultTiDBControl{httpClient: &http.Client{Timeout: timeout}}
+}
+
+func (tdc *defaultTiDBControl) useTLSHTTPClient(enableTLS bool) error {
+	if enableTLS {
+		rootCAs, err := httputil.ReadCACerts()
+		if err != nil {
+			return err
+		}
+		config := &tls.Config{
+			RootCAs: rootCAs,
+		}
+		tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config}
+	}
+	return nil
 }
 
 func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bool {
 	tcName := tc.GetName()
 	ns := tc.GetNamespace()
+	scheme := tc.Scheme()
 
 	result := map[string]bool{}
+
+	if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil {
+		return result
+	}
+
 	for i := 0; i < int(tc.TiDBRealReplicas()); i++ {
 		hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), i)
-		url := fmt.Sprintf("http://%s.%s.%s:10080/status", hostName, TiDBPeerMemberName(tcName), ns)
+		url := fmt.Sprintf("%s://%s.%s.%s:10080/status", scheme, hostName, TiDBPeerMemberName(tcName), ns)
 		_, err := tdc.getBodyOK(url)
 		if err != nil {
 			result[hostName] = false
@@ -80,9 +100,13 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo
 func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) {
 	tcName := tc.GetName()
 	ns := tc.GetNamespace()
+	scheme := tc.Scheme()
+	if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil {
+		return false, err
+	}
 
 	hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal)
-	url := fmt.Sprintf("http://%s.%s.%s:10080/ddl/owner/resign", hostName, TiDBPeerMemberName(tcName), ns)
+	url := fmt.Sprintf("%s://%s.%s.%s:10080/ddl/owner/resign", scheme, hostName, TiDBPeerMemberName(tcName), ns)
 	req, err := http.NewRequest("POST", url, nil)
 	if err != nil {
 		return false, err
@@ -105,9 +129,13 @@ func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal
 func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) (*dbInfo, error) {
 	tcName := tc.GetName()
 	ns := tc.GetNamespace()
+	scheme := tc.Scheme()
+	if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil {
+		return nil, err
+	}
 
 	hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal)
-	url := fmt.Sprintf("http://%s.%s.%s:10080/info", hostName, TiDBPeerMemberName(tcName), ns)
+	url := fmt.Sprintf("%s://%s.%s.%s:10080/info", scheme, hostName, TiDBPeerMemberName(tcName), ns)
 	req, err := http.NewRequest("POST", url, nil)
 	if err != nil {
 		return nil, err
@@ -136,9 +164,13 @@ func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32)
 func (tdc *defaultTiDBControl) GetSettings(tc *v1alpha1.TidbCluster, ordinal int32) (*config.Config, error) {
 	tcName := tc.GetName()
 	ns := tc.GetNamespace()
+	scheme := tc.Scheme()
+	if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil {
+		return nil, err
+	}
 
 	hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal)
-	url := fmt.Sprintf("http://%s.%s.%s:10080/settings", hostName, TiDBPeerMemberName(tcName), ns)
+	url := fmt.Sprintf("%s://%s.%s.%s:10080/settings", scheme, hostName, TiDBPeerMemberName(tcName), ns)
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, err

pkg/discovery/discovery.go

@@ -94,10 +94,10 @@ func (td *tidbDiscovery) Discover(advertisePeerUrl string) (string, error) {
 
 	if len(currentCluster.peers) == int(replicas) {
 		delete(currentCluster.peers, podName)
-		return fmt.Sprintf("--initial-cluster=%s=http://%s", podName, advertisePeerUrl), nil
+		return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, tc.Scheme(), advertisePeerUrl), nil
 	}
 
-	pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName())
+	pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster)
 	membersInfo, err := pdClient.GetMembers()
 	if err != nil {
 		return "", err