From 108e54c17c7e4a119522d838da49295325b23b81 Mon Sep 17 00:00:00 2001 From: Jon Tirsen Date: Fri, 28 Feb 2020 15:23:05 +1100 Subject: [PATCH 1/4] Support no secret for s3/ceph This is required if you use EKS ServiceAccount -> IAM role authentication via OIDC. --- charts/tidb-cluster/templates/scheduled-backup-cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/tidb-cluster/templates/scheduled-backup-cronjob.yaml b/charts/tidb-cluster/templates/scheduled-backup-cronjob.yaml index 5b8fba14861..d99b88ace83 100644 --- a/charts/tidb-cluster/templates/scheduled-backup-cronjob.yaml +++ b/charts/tidb-cluster/templates/scheduled-backup-cronjob.yaml @@ -74,7 +74,7 @@ spec: - name: GOOGLE_APPLICATION_CREDENTIALS value: /gcp/credentials.json {{- end }} - {{- if or .Values.scheduledBackup.ceph .Values.scheduledBackup.s3 }} + {{- if or .Values.scheduledBackup.ceph.secretName .Values.scheduledBackup.s3.secretName }} - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: From 6729b99534105aaae6b8ce2f49a62f812b87e503 Mon Sep 17 00:00:00 2001 From: Jon Tirsen Date: Fri, 28 Feb 2020 16:54:51 +1100 Subject: [PATCH 2/4] Use the environment directly for AWS credentials for rclone --- .../templates/scripts/_start_backup.sh.tpl | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl index 8be218a3efe..e0afae06fad 100644 --- a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl +++ b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl @@ -89,13 +89,24 @@ uploader \ {{- end }} {{- if .Values.s3 }} -uploader \ - --cloud=aws \ - --region={{ .Values.s3.region }} \ - {{- if .Values.s3.prefix }} - --bucket={{ .Values.s3.bucket }}/{{ .Values.s3.prefix }} \ - {{- else }} - --bucket={{ .Values.s3.bucket }} \ - {{- end }} - --backup-dir=${dirname} +# Once we know there are no more credentials that will be logged we can run with -x +set -x +bucket={{ .Values.s3.bucket }} + +cat < /tmp/rclone.conf +[aws] +type = s3 +provider = AWS +env_auth = true +region = us-west-2 +EOF + +cd "${backup_base_dir}" +{{- if .Values.s3.prefix }} +tar -cf - "${backup_name}" | pigz -p 16 \ + | rclone --config /tmp/rclone.conf rcat s3:${bucket}/{{ .Values.s3.prefix }}/${backup_name}/${backup_name}.tgz +{{- else }} +tar -cf - "${backup_name}" | pigz -p 16 \ + | rclone --config /tmp/rclone.conf rcat s3:${bucket}/${backup_name}/${backup_name}.tgz +{{- end }} {{- end }} From 6d7e1cdf73db943edf3248aa6e290edc6e1083b4 Mon Sep 17 00:00:00 2001 From: Jon Tirsen Date: Sat, 29 Feb 2020 15:57:59 +1100 Subject: [PATCH 3/4] Fixes to backup scripts --- .../templates/scripts/_start_backup.sh.tpl | 4 +- .../scripts/_start_scheduled_backup.sh.tpl | 55 +++++++++++++++---- charts/tidb-cluster/values.yaml | 2 + 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl index e0afae06fad..b321c6a52dc 100644 --- a/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl +++ b/charts/tidb-backup/templates/scripts/_start_backup.sh.tpl @@ -94,11 +94,11 @@ set -x bucket={{ .Values.s3.bucket }} cat < /tmp/rclone.conf -[aws] +[s3] type = s3 provider = AWS env_auth = true -region = us-west-2 +region = {{ .Values.s3.region }} EOF cd "${backup_base_dir}" diff --git a/charts/tidb-cluster/templates/scripts/_start_scheduled_backup.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_scheduled_backup.sh.tpl index 3e42ef70911..99017564758 100755 --- a/charts/tidb-cluster/templates/scripts/_start_scheduled_backup.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_scheduled_backup.sh.tpl @@ -3,7 +3,8 @@ set -euo pipefail host=$(getent hosts {{ template "cluster.name" . }}-tidb | head | awk '{print $1}') backupName=scheduled-backup-`date "+%Y%m%d-%H%M%S"` -backupPath=/data/${backupName} +backupBase=/data +backupPath=${backupBase}/${backupName} echo "making dir ${backupPath}" mkdir -p ${backupPath} @@ -37,10 +38,29 @@ echo "Reset TiKV GC life time to ${gc_life_time}" /usr/bin/mysql -h${host} -P4000 -u${TIDB_USER} ${password_str} -Nse "select variable_name,variable_value from mysql.tidb where variable_name='tikv_gc_life_time';" {{- if .Values.scheduledBackup.gcp }} -uploader \ - --cloud=gcp \ - --bucket={{ .Values.scheduledBackup.gcp.bucket }} \ - --backup-dir=${backupPath} +# Once we know there are no more credentials that will be logged we can run with -x +set -x +bucket={{ .Values.scheduledBackup.gcp.bucket }} +creds=${GOOGLE_APPLICATION_CREDENTIALS:-""} +if ! [[ -z $creds ]] ; then +creds="service_account_file = ${creds}" +fi + +cat < /tmp/rclone.conf +[gcp] +type = google cloud storage +bucket_policy_only = true +$creds +EOF + +cd "${backupBase}" +{{- if .Values.scheduledBackup.gcp.prefix }} +tar -cf - "${backupName}" | pigz -p 16 \ + | rclone --config /tmp/rclone.conf rcat gcp:${bucket}/{{ .Values.scheduledBackup.gcp.prefix }}/${backupName}/${backupName}.tgz +{{- else }} +tar -cf - "${backupName}" | pigz -p 16 \ + | rclone --config /tmp/rclone.conf rcat gcp:${bucket}/${backupName}/${backupName}.tgz +{{- end }} {{- end }} {{- if .Values.scheduledBackup.ceph }} @@ -52,11 +72,26 @@ uploader \ {{- end }} {{- if .Values.scheduledBackup.s3 }} -uploader \ - --cloud=aws \ - --region={{ .Values.scheduledBackup.s3.region }} \ - --bucket={{ .Values.scheduledBackup.s3.bucket }} \ - --backup-dir=${backupPath} +# Once we know there are no more credentials that will be logged we can run with -x +set -x +bucket={{ .Values.scheduledBackup.s3.bucket }} + +cat < /tmp/rclone.conf +[s3] +type = s3 +provider = AWS +env_auth = true +region = {{ .Values.scheduledBackup.s3.region }} +EOF + +cd "${backupBase}" +{{- if .Values.scheduledBackup.s3.prefix }} +tar -cf - "${backupName}" | pigz -p 16 \ + | rclone --config /tmp/rclone.conf rcat s3:${bucket}/{{ .Values.scheduledBackup.s3.prefix }}/${backupName}/${backupName}.tgz +{{- else }} +tar -cf - "${backupName}" | pigz -p 16 \ + | rclone --config /tmp/rclone.conf rcat s3:${bucket}/${backupName}/${backupName}.tgz +{{- end }} {{- end }} {{- if and (.Values.scheduledBackup.cleanupAfterUpload) (or (.Values.scheduledBackup.gcp) (or .Values.scheduledBackup.ceph .Values.scheduledBackup.s3)) }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index e972d8d08ff..4afc2135d8e 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -741,6 +741,7 @@ scheduledBackup: # backup to gcp gcp: {} # bucket: "" + # prefix: "" # secretName is the name of the secret which stores the gcp service account credentials json file # The service account must have read/write permission to the above bucket. # Read the following document to create the service account and download the credentials file as credentials.json: @@ -761,6 +762,7 @@ scheduledBackup: s3: {} # region: "" # bucket: "" + # prefix: "" # secretName is the name of the secret which stores s3 object store access key and secret key # You can create the secret by: # kubectl create secret generic s3-backup-secret --from-literal=access_key= --from-literal=secret_key= From 68dd370751d28327671c29c0e6cb55a3e83f1313 Mon Sep 17 00:00:00 2001 From: Jon Tirsen Date: Mon, 2 Mar 2020 10:42:44 +1100 Subject: [PATCH 4/4] Update backup image to `pingcap/tidb-cloud-backup:20200229` --- charts/tidb-backup/values.yaml | 2 +- charts/tidb-cluster/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/tidb-backup/values.yaml b/charts/tidb-backup/values.yaml index b1268558213..14a1044a378 100644 --- a/charts/tidb-backup/values.yaml +++ b/charts/tidb-backup/values.yaml @@ -27,7 +27,7 @@ name: fullbackup-{{ date "200601021504" .Release.Time }} image: pullPolicy: IfNotPresent # https://github.com/pingcap/tidb-cloud-backup - backup: pingcap/tidb-cloud-backup:20191217 + backup: pingcap/tidb-cloud-backup:20200229 ## nodeSelector ensure pods only assigning to nodes which have each of the indicated key-value pairs as labels ## ref:https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 4afc2135d8e..184226bc2e0 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -700,7 +700,7 @@ binlog: scheduledBackup: create: false # https://github.com/pingcap/tidb-cloud-backup - mydumperImage: pingcap/tidb-cloud-backup:20191217 + mydumperImage: pingcap/tidb-cloud-backup:20200229 mydumperImagePullPolicy: IfNotPresent # storageClassName is a StorageClass provides a way for administrators to describe the "classes" of storage they offer. # different classes might map to quality-of-service levels, or to backup policies,