Merge branch 'kind-cluster' of https://github.com/xiaojingchen/tidb-o…

…perator into kind-cluster

charts/tidb-backup/templates/backup-job.yaml

@@ -45,7 +45,7 @@ spec:
       {{- end }}
         env:
         - name: BACKUP_NAME
-          value: {{ .Values.name | quote }}
+          value: {{ tpl .Values.name . | quote }}
       {{- if .Values.gcp }}
         - name: GOOGLE_APPLICATION_CREDENTIALS
           value: /gcp/credentials.json
@@ -76,7 +76,7 @@ spec:
       volumes:
       - name: data
         persistentVolumeClaim:
-          claimName: {{ .Values.name }}
+          claimName: {{ tpl .Values.name . }}
     {{- if .Values.gcp }}
       - name: gcp-credentials
         secret:

charts/tidb-backup/templates/scripts/_start_backup.sh.tpl

@@ -15,8 +15,8 @@ fi
 gc_life_time=`/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "select variable_value from mysql.tidb where variable_name='tikv_gc_life_time';"`
 echo "Old TiKV GC life time is ${gc_life_time}"
 
-echo "Increase TiKV GC life time to 3h"
-/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "update mysql.tidb set variable_value='3h' where variable_name='tikv_gc_life_time';"
+echo "Increase TiKV GC life time to {{ .Values.tikvGCLifeTime | default "720h" }}"
+/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "update mysql.tidb set variable_value='{{ .Values.tikvGCLifeTime | default "720h" }}' where variable_name='tikv_gc_life_time';"
 /usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "select variable_name,variable_value from mysql.tidb where variable_name='tikv_gc_life_time';"
 
 if [ -n "{{ .Values.initialCommitTs }}" ];

charts/tidb-backup/values.yaml

@@ -17,7 +17,7 @@ name: fullbackup-{{ date "200601021504" .Release.Time }}
 image:
   pullPolicy: IfNotPresent
   # https://github.com/pingcap/tidb-cloud-backup
-  backup: pingcap/tidb-cloud-backup:20190610
+  backup: pingcap/tidb-cloud-backup:20190828
 
 # Add additional labels for backup/restore job's pod
 # ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
@@ -41,8 +41,11 @@ storage:
   className: local-storage
   size: 100Gi
 
-# backupOptions is the options of mydumper https://github.com/maxbube/mydumper/blob/master/docs/mydumper_usage.rst#options
-backupOptions: "--verbose=3"
+# -t is thread count, larger thread count will speed up the backup, but may impact the performance of the upstream TiDB.
+# -F is the chunk size, a big table is partitioned into many chunks.
+# Other useful options are -B for database, and -T for tables.
+# See https://github.com/maxbube/mydumper/blob/master/docs/mydumper_usage.rst#options for more options.
+backupOptions: "-t 16 -F 256 --skip-tz-utc --verbose=3"
 # Set the tidb_snapshot to be used for the backup
 # Use `show master status` to get the ts:
 #   MySQL [(none)]> show master status;
@@ -56,6 +59,9 @@ backupOptions: "--verbose=3"
 initialCommitTs: ""
 # restoreOptions is the options of loader https://www.pingcap.com/docs-cn/tools/loader/
 restoreOptions: "-t 16"
+# The time limit during which data is retained for each GC when backup, in the format of Go Duration.
+# When a GC happens, the current time minus this value is the safe point.
+tikvGCLifeTime: 720h
 
 # By default, the backup/restore uses PV to store/load backup data
 # You can choose to store/load backup data to/from gcp, ceph or s3 bucket by enabling the following corresponding section:

charts/tidb-cluster/templates/config/_drainer-config.tpl

@@ -46,7 +46,7 @@ disable-dispatch = {{ .Values.binlog.drainer.disableDispatch | default false }}
 safe-mode = {{ .Values.binlog.drainer.safeMode | default false }}
 
 # downstream storage, equal to --dest-db-type
-# valid values are "mysql", "pb", "tidb", "flash", "kafka"
+# valid values are "mysql", "file", "tidb", "flash", "kafka"
 db-type = "{{ .Values.binlog.drainer.destDBType }}"
 
 # disable sync these schema
@@ -70,7 +70,7 @@ ignore-schemas = {{ .Values.binlog.drainer.ignoreSchemas | default "INFORMATION_
 #db-name = "test"
 #tbl-name = "log"
 
-{{- if eq .Values.binlog.drainer.destDBType "mysql" }}
+{{- if or (eq .Values.binlog.drainer.destDBType "mysql") (eq .Values.binlog.drainer.destDBType "tidb") }}
 # the downstream mysql protocol database
 [syncer.to]
 host = {{ .Values.binlog.drainer.mysql.host | quote }}
@@ -83,14 +83,11 @@ port = {{ .Values.binlog.drainer.mysql.port | default 3306 }}
 #schema = "tidb_binlog"
 {{- end }}
 
-{{- if eq .Values.binlog.drainer.destDBType "pb" }}
-# Uncomment this if you want to use pb or sql as db-type.
-# Compress compresses output file, like pb and sql file. Now it supports "gzip" algorithm only.
-# Values can be "gzip". Leave it empty to disable compression.
+{{- if or (eq .Values.binlog.drainer.destDBType "pb") (eq .Values.binlog.drainer.destDBType "file") }}
+# Uncomment this if you want to use file as db-type.
 [syncer.to]
-# directory to save pb file, default same as data-dir(save checkpoint file) if this is not configured.
+# directory to save binlog file, default same as data-dir(save checkpoint file) if this is not configured.
 dir = "/data/pb"
-compression = "gzip"
 {{- end }}
 
 {{- if eq .Values.binlog.drainer.destDBType "kafka" }}

charts/tidb-cluster/templates/scripts/_start_scheduled_backup.sh.tpl

@@ -19,8 +19,8 @@ fi
 gc_life_time=`/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "select variable_value from mysql.tidb where variable_name='tikv_gc_life_time';"`
 echo "Old TiKV GC life time is ${gc_life_time}"
 
-echo "Increase TiKV GC life time to 3h"
-/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "update mysql.tidb set variable_value='3h' where variable_name='tikv_gc_life_time';"
+echo "Increase TiKV GC life time to {{ .Values.scheduledBackup.tikvGCLifeTime | default "720h" }}"
+/usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "update mysql.tidb set variable_value='{{ .Values.scheduledBackup.tikvGCLifeTime | default "720h" }}' where variable_name='tikv_gc_life_time';"
 /usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "select variable_name,variable_value from mysql.tidb where variable_name='tikv_gc_life_time';"
 
 /mydumper \

charts/tidb-cluster/values.yaml

@@ -465,8 +465,8 @@ binlog:
     # the number of SQL statements of a transaction that are output to the downstream database (20 by default)
     txnBatch: 20
     # downstream storage, equal to --dest-db-type
-    # valid values are "mysql", "pb", "kafka"
-    destDBType: pb
+    # valid values are "mysql", "file", "tidb", "kafka"
+    destDBType: file
     mysql: {}
       # host: "127.0.0.1"
       # user: "root"
@@ -502,7 +502,7 @@ binlog:
 scheduledBackup:
   create: false
   # https://github.com/pingcap/tidb-cloud-backup
-  mydumperImage: pingcap/tidb-cloud-backup:20190610
+  mydumperImage: pingcap/tidb-cloud-backup:20190828
   mydumperImagePullPolicy: IfNotPresent
   # storageClassName is a StorageClass provides a way for administrators to describe the "classes" of storage they offer.
   # different classes might map to quality-of-service levels, or to backup policies,
@@ -519,8 +519,14 @@ scheduledBackup:
   failedJobsHistoryLimit: 1
   # https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#starting-deadline
   startingDeadlineSeconds: 3600
-  # https://github.com/maxbube/mydumper/blob/master/docs/mydumper_usage.rst#options
-  options: "--verbose=3"
+  # -t is thread count, larger thread count will speed up the backup, but may impact the performance of the upstream TiDB.
+  # -F is the chunk size, a big table is partitioned into many chunks.
+  # Other useful options are -B for database, and -T for tables.
+  # See https://github.com/maxbube/mydumper/blob/master/docs/mydumper_usage.rst#options for more options.
+  options: "-t 16 -F 256 --skip-tz-utc --verbose=3"
+  # The time limit during which data is retained for each GC when backup, in the format of Go Duration.
+  # When a GC happens, the current time minus this value is the safe point.
+  tikvGCLifeTime: 720h
   # secretName is the name of the secret which stores user and password used for backup
   # Note: you must give the user enough privilege to do the backup
   # you can create the secret by:

charts/tidb-lightning/templates/job.yaml

@@ -23,7 +23,7 @@ spec:
     spec:
       {{ if and .Values.dataSource.local.hostPath .Values.dataSource.local.nodeName -}}
       nodeName: {{ .Values.dataSource.local.nodeName }}
-      {{ else -}}
+      {{ else if not .Values.dataSource.adhoc.pvcName -}}
       initContainers:
       - name: data-retriever
         image: {{ .Values.dataSource.remote.rcloneImage }}
@@ -78,7 +78,11 @@ spec:
         hostPath:
           path: {{ .Values.dataSource.local.hostPath }}
           type: Directory
-      {{- else -}}
+      {{- else if .Values.dataSource.adhoc.pvcName -}}
+      - name: data
+        persistentVolumeClaim:
+          claimName: {{ .Values.dataSource.adhoc.pvcName }}
+      {{ else }}
       - name: credentials
         secret:
           secretName: {{ .Values.dataSource.remote.secretName }}

charts/tidb-lightning/templates/persistentvolumeclaim.yaml

@@ -1,3 +1,5 @@
+{{ if or .Values.dataSource.adhoc.pvcName (and .Values.dataSource.local.nodeName .Values.dataSource.local.hostPath) }}
+{{- else -}}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -15,3 +17,4 @@ spec:
   resources:
     requests:
       storage: {{ .Values.dataSource.remote.storage }}
+{{ end -}}

charts/tidb-lightning/templates/scripts/_start_lightning.sh.tpl

@@ -1,5 +1,7 @@
 {{- if and .Values.dataSource.local.hostPath .Values.dataSource.local.nodeName -}}
 data_dir={{ .Values.dataSource.local.hostPath }}
+{{- else if .Values.dataSource.adhoc.pvcName -}}
+data_dir=/var/lib/tidb-lightning/{{ .Values.dataSource.adhoc.backupName | default .Values.dataSource.adhoc.pvcName }}
 {{- else -}}
 data_dir=$(dirname $(find /var/lib/tidb-lightning -name metadata 2>/dev/null) 2>/dev/null)
 if [ -z $data_dir ]; then
@@ -12,12 +14,12 @@ if [ -z $data_dir ]; then
 fi
 {{ end }}
 /tidb-lightning \
-    --pd-urls={{ .Values.targetTidbCluster.name }}-pd.{{ .Release.Namespace }}:2379 \
+    --pd-urls={{ .Values.targetTidbCluster.name }}-pd.{{ .Values.targetTidbCluster.namespace | default .Release.Namespace }}:2379 \
     --status-addr=0.0.0.0:8289 \
-    --importer={{ .Values.targetTidbCluster.name }}-importer.{{ .Release.Namespace }}:8287 \
+    --importer={{ .Values.targetTidbCluster.name }}-importer.{{ .Values.targetTidbCluster.namespace | default .Release.Namespace }}:8287 \
     --server-mode=false \
     --tidb-user={{ .Values.targetTidbCluster.user | default "root" }} \
-    --tidb-host={{ .Values.targetTidbCluster.name }}-tidb.{{ .Release.Namespace }} \
+    --tidb-host={{ .Values.targetTidbCluster.name }}-tidb.{{ .Values.targetTidbCluster.namespace | default .Release.Namespace }} \
     --d=${data_dir} \
     --config=/etc/tidb-lightning/tidb-lightning.toml
 

charts/tidb-lightning/values.yaml

@@ -9,12 +9,18 @@ service:
 
 # failFast causes the lightning pod fails when any error happens.
 # when disabled, the lightning pod will keep running when error happens to allow manual intervention, users have to check logs to see the job status.
-failFast: false
+failFast: true
 
 dataSource:
   local: {}
     # nodeName: kind-worker3
     # hostPath: /data/export-20190820
+  # The backup data is on a PVC which is from tidb-backup or scheduled backup, and is not uploaded to cloud storage yet.
+  # Note: when using this mode, the lightning needs to be deployed in the same namespace as the PVC
+  # and the `targetTidbCluster.namespace` needs to be configured explicitly
+  adhoc: {}
+    # pvcName: tidb-cluster-scheduled-backup
+    # backupName: scheduled-backup-20190822-041004
   remote:
     rcloneImage: tynor88/rclone
     storageClassName: local-storage
@@ -24,6 +30,8 @@ dataSource:
 
 targetTidbCluster:
   name: tidb-cluster
+  # namespace is the target tidb cluster namespace, can be omitted if the lightning is deployed in the same namespace of the target tidb cluster
+  namespace: ""
   user: root
 
 resources: {}

cmd/admission-controller/main.go

@@ -21,8 +21,8 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
+	"github.com/pingcap/tidb-operator/pkg/version"
 	"github.com/pingcap/tidb-operator/pkg/webhook"
-	"github.com/pingcap/tidb-operator/version"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"

cmd/controller-manager/main.go

@@ -29,7 +29,7 @@ import (
 	"github.com/pingcap/tidb-operator/pkg/controller/backupschedule"
 	"github.com/pingcap/tidb-operator/pkg/controller/restore"
 	"github.com/pingcap/tidb-operator/pkg/controller/tidbcluster"
-	"github.com/pingcap/tidb-operator/version"
+	"github.com/pingcap/tidb-operator/pkg/version"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/util/logs"

cmd/discovery/main.go

@@ -10,7 +10,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/discovery/server"
-	"github.com/pingcap/tidb-operator/version"
+	"github.com/pingcap/tidb-operator/pkg/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/client-go/rest"

cmd/scheduler/main.go

@@ -24,7 +24,7 @@ import (
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/features"
 	"github.com/pingcap/tidb-operator/pkg/scheduler/server"
-	"github.com/pingcap/tidb-operator/version"
+	"github.com/pingcap/tidb-operator/pkg/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/client-go/kubernetes"

deploy/gcp/tidbclusters.tf

@@ -1,32 +1,42 @@
 data "local_file" "kubeconfig" {
-  depends_on = [module.tidb-operator.cluster_id]
-  filename   = module.tidb-operator.kubeconfig_path
+  depends_on = [module.tidb-operator.get_credentials_id]
+  filename   = local.kubeconfig
 }
+
+# local file resource used to delay helm provider initialization
 resource "local_file" "kubeconfig" {
-  depends_on = [module.tidb-operator.cluster_id]
   content    = data.local_file.kubeconfig.content
-  filename   = module.tidb-operator.kubeconfig_path
+  filename   = local.kubeconfig
 }
 
 provider "helm" {
   alias          = "gke"
   insecure       = true
   install_tiller = false
   kubernetes {
-    config_path = local_file.kubeconfig.filename
+    # helm provider loads the file when it's initialized, we must wait for it to be created.
+    # However we cannot use resource here, because in refresh phrase, it will
+    # not be resolved and argument default value is used. To work around this,
+    # we defer initialization by using load_config_file argument.
+    # See https://github.com/pingcap/tidb-operator/pull/819#issuecomment-524547459
+    config_path = local.kubeconfig
+    load_config_file = local_file.kubeconfig.filename != "" ? true : false
   }
 }
+
 module "default-tidb-cluster" {
   providers = {
     helm = "helm.gke"
   }
   source                     = "../modules/gcp/tidb-cluster"
-  gcp_project                = module.tidb-operator.gcp_project
-  gke_cluster_location       = module.tidb-operator.gke_cluster_location
-  gke_cluster_name           = module.tidb-operator.gke_cluster_name
+  cluster_id                 = module.tidb-operator.cluster_id
+  tidb_operator_id           = module.tidb-operator.tidb_operator_id
+  gcp_project                = var.GCP_PROJECT
+  gke_cluster_location       = local.location
+  gke_cluster_name           = var.gke_name
   cluster_name               = var.default_tidb_cluster_name
   cluster_version            = var.tidb_version
-  kubeconfig_path            = module.tidb-operator.kubeconfig_path
+  kubeconfig_path            = local.kubeconfig
   tidb_cluster_chart_version = coalesce(var.tidb_operator_chart_version, var.tidb_operator_version)
   pd_instance_type           = var.pd_instance_type
   tikv_instance_type         = var.tikv_instance_type

deploy/modules/gcp/tidb-cluster/data.tf

@@ -23,10 +23,11 @@ locals {
   cluster_location_args = "%{if length(split("-", var.gke_cluster_location)) == 3}--zone ${var.gke_cluster_location} %{else}--region ${var.gke_cluster_location} %{endif}"
   # TODO Update related code when node locations is avaiable in attributes of cluster resource.
   cmd_get_cluster_locations = <<EOT
-gcloud --project ${var.gcp_project} container clusters list --filter='name=${var.gke_cluster_name}' --format='json[no-heading](locations)' ${local.cluster_location_args} | jq '.[0] | .locations |= join(",")'
+gcloud --project ${var.gcp_project} container clusters list --filter='name=${var.gke_cluster_name}' --format='json[no-heading](locations)' ${local.cluster_location_args} | jq '{"locations": (if (. | length) > 0 then .[0].locations | join(",") else "" end) }'
 EOT
 }
 
 data "external" "cluster_locations" {
+  depends_on = [var.cluster_id]
   program = ["bash", "-c", local.cmd_get_cluster_locations]
 }

deploy/modules/gcp/tidb-cluster/main.tf

@@ -41,6 +41,11 @@ resource "google_container_node_pool" "tikv_pool" {
   name       = "${var.cluster_name}-tikv-pool"
   node_count = var.tikv_node_count
 
+  # tikv_pool is the first resource of node pools to create in this module, wait for the cluster to be ready
+  depends_on = [
+    var.cluster_id
+  ]
+
   management {
     auto_repair  = false
     auto_upgrade = false
@@ -139,7 +144,7 @@ module "tidb-cluster" {
   override_values            = var.override_values
   kubeconfig_filename        = var.kubeconfig_path
   base_values                = file("${path.module}/values/default.yaml")
-  wait_on_resource           = [google_container_node_pool.tidb_pool]
+  wait_on_resource           = [google_container_node_pool.tidb_pool, var.tidb_operator_id]
 }
 
 resource "null_resource" "wait-lb-ip" {

deploy/modules/gcp/tidb-cluster/variables.tf

@@ -1,3 +1,11 @@
+variable "cluster_id" {
+  description = "GKE cluster ID. This module depends on a running cluster. Please create a cluster first and pass ID here."
+}
+
+variable "tidb_operator_id" {
+  description = "TiDB Operator ID. We must wait for tidb-operator is ready before creating TiDB clusters."
+}
+
 variable "cluster_name" {}
 variable "cluster_version" {
   description = "The TiDB cluster version"

deploy/modules/gcp/tidb-operator/main.tf

@@ -66,12 +66,12 @@ resource "null_resource" "get-credentials" {
 }
 
 data "local_file" "kubeconfig" {
-  depends_on = [google_container_cluster.cluster, null_resource.get-credentials]
+  depends_on = [null_resource.get-credentials]
   filename   = var.kubeconfig_path
 }
 
+# local file resource used to delay helm provider initialization
 resource "local_file" "kubeconfig" {
-  depends_on = [google_container_cluster.cluster, null_resource.get-credentials]
   content    = data.local_file.kubeconfig.content
   filename   = var.kubeconfig_path
 }
@@ -82,7 +82,13 @@ provider "helm" {
   # service_account = "tiller"
   install_tiller = false # currently this doesn't work, so we install tiller in the local-exec provisioner. See https://github.com/terraform-providers/terraform-provider-helm/issues/148
   kubernetes {
-    config_path = local_file.kubeconfig.filename
+    # helm provider loads the file when it's initialized, we must wait for it to be created.
+    # However we cannot use resource here, because in refresh phrase, it will
+    # not be resolved and argument default value is used. To work around this,
+    # we defer initialization by using load_config_file argument.
+    # See https://github.com/pingcap/tidb-operator/pull/819#issuecomment-524547459
+    config_path = var.kubeconfig_path
+    load_config_file = local_file.kubeconfig.filename != "" ? true : false
   }
 }