From cf78457080989524efa446af12a7f783ce4fc39d Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Wed, 23 Jun 2021 19:47:16 +0800
Subject: [PATCH 01/12] add test

---
 tests/_utils/run_services            |   3 +-
 tests/br_restore_TDE_enable/run.sh   | 126 +++++++++++++++++++++++++++
 tests/br_restore_TDE_enable/workload |  12 +++
 tests/config/tikv.toml               |   7 ++
 4 files changed, 147 insertions(+), 1 deletion(-)
 create mode 100755 tests/br_restore_TDE_enable/run.sh
 create mode 100644 tests/br_restore_TDE_enable/workload

diff --git a/tests/_utils/run_services b/tests/_utils/run_services
index ed568a955..95e03b06b 100644
--- a/tests/_utils/run_services
+++ b/tests/_utils/run_services
@@ -140,7 +140,7 @@ start_tikv() {
         --log-file "$TEST_DIR/tikv${i}.log" \
         --log-level info \
         -C "$TIKV_CONFIG" \
-        -s "$TEST_DIR/tikv${i}" &
+        -s "$TEST_DIR/tikv${i}" 2> /tmp/backup_restore_test/stderr.log &
     pid=$!
     echo -e "$pid\t$TIKV_CONFIG" > "${TIKV_PIDS}_${i}"
 }
@@ -223,6 +223,7 @@ start_services_impl() {
     rm -f "${TIKV_PIDS}*"
 
     start_pd
+    echo -e "3b5896b5be691006e0f71c3040a29495ddcad20b14aff61806940ebd780d3c62" > "/tmp/file"
     for i in $(seq $TIKV_COUNT); do
         start_tikv "$i"
     done
diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
new file mode 100755
index 000000000..d271fca0d
--- /dev/null
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+#
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+DB="$TEST_NAME"
+TABLE="usertable"
+DB_COUNT=3
+
+# start the s3 server
+
+curl -sSL --tlsv1.2 \
+     -O 'https://raw.githubusercontent.com/minio/kes/master/root.key' \
+     -O 'https://raw.githubusercontent.com/minio/kes/master/root.cert'
+
+export MINIO_KMS_KES_ENDPOINT=https://play.min.io:7373
+export MINIO_KMS_KES_KEY_FILE=root.key
+export MINIO_KMS_KES_CERT_FILE=root.cert
+export MINIO_KMS_KES_KEY_NAME=my-minio-key
+
+export MINIO_ACCESS_KEY='KEXI7MANNASOPDLAOIEF'
+export MINIO_SECRET_KEY='MaKYxEGDInMPtEYECXRJLU+FPNKb/wAX/MElir7E'
+export MINIO_BROWSER=off
+export AWS_ACCESS_KEY_ID=$MINIO_ACCESS_KEY
+export AWS_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
+export S3_ENDPOINT=127.0.0.1:24927
+
+rm -rf "$TEST_DIR/$DB"
+mkdir -p "$TEST_DIR/$DB"
+
+start_s3() {
+    bin/minio server --address $S3_ENDPOINT "$TEST_DIR/$DB" &
+    s3_pid=$!
+    i=0
+    while ! curl -o /dev/null -v -s "http://$S3_ENDPOINT/"; do
+        i=$(($i+1))
+        if [ $i -gt 30 ]; then
+            echo 'Failed to start minio'
+            exit 1
+        fi
+        sleep 2
+    done
+}
+
+start_s3
+echo "started s3 with pid = $s3_pid"
+
+
+
+bin/mc config --config-dir "$TEST_DIR/$TEST_NAME" \
+    host add minio http://$S3_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
+
+# Fill in the database
+for i in $(seq $DB_COUNT); do
+    run_sql "CREATE DATABASE $DB${i};"
+    go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
+done
+
+bin/mc mb --config-dir "$TEST_DIR/$TEST_NAME" minio/mybucket
+S3_KEY=""
+for p in $(seq 2); do
+
+  for i in $(seq $DB_COUNT); do
+      row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
+  done
+
+  # backup full
+  echo "backup start..."
+  BACKUP_LOG="backup.log"
+  rm -f $BACKUP_LOG
+  unset BR_LOG_TO_TERM
+  run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT$S3_KEY" \
+      --log-file $BACKUP_LOG \
+      --s3.sse AES256
+
+  for i in $(seq $DB_COUNT); do
+      run_sql "DROP DATABASE $DB${i};"
+  done
+
+  # restore full
+  echo "restore start..."
+  RESTORE_LOG="restore.log"
+  rm -f $RESTORE_LOG
+  unset BR_LOG_TO_TERM
+  run_br restore full -s "s3://mybucket/$DB?$S3_KEY" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \
+      --log-file $RESTORE_LOG 
+
+  for i in $(seq $DB_COUNT); do
+      row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
+  done
+
+  fail=false
+  for i in $(seq $DB_COUNT); do
+      if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then
+          fail=true
+          echo "TEST: [$TEST_NAME] fail on database $DB${i}"
+      fi
+      echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}"
+  done
+
+  if $fail; then
+      echo "TEST: [$TEST_NAME] failed!"
+      exit 1
+  fi
+
+  # prepare for next test
+  bin/mc rm --config-dir "$TEST_DIR/$TEST_NAME" --recursive --force minio/mybucket
+  S3_KEY="&access-key=$MINIO_ACCESS_KEY&secret-access-key=$MINIO_SECRET_KEY"
+  export AWS_ACCESS_KEY_ID=""
+  export AWS_SECRET_ACCESS_KEY=""
+done
+
+for i in $(seq $DB_COUNT); do
+    run_sql "DROP DATABASE $DB${i};"
+done
diff --git a/tests/br_restore_TDE_enable/workload b/tests/br_restore_TDE_enable/workload
new file mode 100644
index 000000000..e3fadf9a3
--- /dev/null
+++ b/tests/br_restore_TDE_enable/workload
@@ -0,0 +1,12 @@
+recordcount=10000
+operationcount=0
+workload=core
+
+readallfields=true
+
+readproportion=0
+updateproportion=0
+scanproportion=0
+insertproportion=0
+
+requestdistribution=uniform
diff --git a/tests/config/tikv.toml b/tests/config/tikv.toml
index b9dec3c66..951ff16bc 100644
--- a/tests/config/tikv.toml
+++ b/tests/config/tikv.toml
@@ -25,3 +25,10 @@ hibernate-regions-compatible=false
 ca-path = "/tmp/backup_restore_test/certs/ca.pem"
 cert-path = "/tmp/backup_restore_test/certs/tikv.pem"
 key-path = "/tmp/backup_restore_test/certs/tikv.key"
+
+[security.encryption]
+data-encryption-method = "aes256-ctr"
+
+[security.encryption.master-key]
+type = "file"
+path = "/tmp/file"

From 9e0bac58209a2bb2c05d2d9ebcb57e03bf3ca10e Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Thu, 24 Jun 2021 12:29:55 +0800
Subject: [PATCH 02/12] add TDE test

---
 tests/_utils/run_services          | 3 ++-
 tests/br_restore_TDE_enable/run.sh | 6 ++++++
 tests/config/tikv.toml             | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tests/_utils/run_services b/tests/_utils/run_services
index 95e03b06b..fa1f3bb06 100644
--- a/tests/_utils/run_services
+++ b/tests/_utils/run_services
@@ -140,7 +140,7 @@ start_tikv() {
         --log-file "$TEST_DIR/tikv${i}.log" \
         --log-level info \
         -C "$TIKV_CONFIG" \
-        -s "$TEST_DIR/tikv${i}" 2> /tmp/backup_restore_test/stderr.log &
+        -s "$TEST_DIR/tikv${i}" &
     pid=$!
     echo -e "$pid\t$TIKV_CONFIG" > "${TIKV_PIDS}_${i}"
 }
@@ -223,6 +223,7 @@ start_services_impl() {
     rm -f "${TIKV_PIDS}*"
 
     start_pd
+    # When using TDE, we add the master key to a file, and this master key is used to encrypt data key
     echo -e "3b5896b5be691006e0f71c3040a29495ddcad20b14aff61806940ebd780d3c62" > "/tmp/file"
     for i in $(seq $TIKV_COUNT); do
         start_tikv "$i"
diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index d271fca0d..113ad572f 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -80,9 +80,15 @@ for p in $(seq 2); do
   BACKUP_LOG="backup.log"
   rm -f $BACKUP_LOG
   unset BR_LOG_TO_TERM
+
+  # using --s3.sse AES256 to ensure backup file are encrypted
   run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT$S3_KEY" \
       --log-file $BACKUP_LOG \
       --s3.sse AES256
+    
+# ensure the tikv data file are encrypted
+bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file --path=/tmp/backup_restore_test/tikv1/db/CURRENT | grep "Aes256Ctr"
+
 
   for i in $(seq $DB_COUNT); do
       run_sql "DROP DATABASE $DB${i};"
diff --git a/tests/config/tikv.toml b/tests/config/tikv.toml
index 951ff16bc..6e832605f 100644
--- a/tests/config/tikv.toml
+++ b/tests/config/tikv.toml
@@ -1,6 +1,7 @@
 # config of tikv
 [storage]
 reserve-space = "1KB"
+data-dir = "/tmp/backup_restore_test/tikv1/"
 
 [coprocessor]
 region-max-keys = 100

From f6fbb3482e8435955c972ea852344ac368f34b9a Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Thu, 24 Jun 2021 12:47:08 +0800
Subject: [PATCH 03/12] add comments

---
 tests/br_restore_TDE_enable/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index 113ad572f..7684712d7 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -18,8 +18,7 @@ DB="$TEST_NAME"
 TABLE="usertable"
 DB_COUNT=3
 
-# start the s3 server
-
+# start Minio KMS service
 curl -sSL --tlsv1.2 \
      -O 'https://raw.githubusercontent.com/minio/kes/master/root.key' \
      -O 'https://raw.githubusercontent.com/minio/kes/master/root.cert'
@@ -29,6 +28,7 @@ export MINIO_KMS_KES_KEY_FILE=root.key
 export MINIO_KMS_KES_CERT_FILE=root.cert
 export MINIO_KMS_KES_KEY_NAME=my-minio-key
 
+# start the s3 server
 export MINIO_ACCESS_KEY='KEXI7MANNASOPDLAOIEF'
 export MINIO_SECRET_KEY='MaKYxEGDInMPtEYECXRJLU+FPNKb/wAX/MElir7E'
 export MINIO_BROWSER=off

From 3db50244ce1b69ed95a1be63193a36d814ce5458 Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Thu, 24 Jun 2021 12:48:16 +0800
Subject: [PATCH 04/12] refomat

---
 tests/br_restore_TDE_enable/run.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index 7684712d7..5bf67327f 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -56,8 +56,6 @@ start_s3() {
 start_s3
 echo "started s3 with pid = $s3_pid"
 
-
-
 bin/mc config --config-dir "$TEST_DIR/$TEST_NAME" \
     host add minio http://$S3_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
 

From 773b1b6d8c5f47a0370a3e163ac6997c734a5553 Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Thu, 1 Jul 2021 15:48:11 +0800
Subject: [PATCH 05/12] fix

---
 tests/br_restore_TDE_enable/run.sh   | 7 ++-----
 tests/br_restore_TDE_enable/workload | 2 +-
 tests/config/root.cert               | 9 +++++++++
 tests/config/root.key                | 3 +++
 4 files changed, 15 insertions(+), 6 deletions(-)
 create mode 100644 tests/config/root.cert
 create mode 100644 tests/config/root.key

diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index 5bf67327f..6beba499f 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -19,13 +19,10 @@ TABLE="usertable"
 DB_COUNT=3
 
 # start Minio KMS service
-curl -sSL --tlsv1.2 \
-     -O 'https://raw.githubusercontent.com/minio/kes/master/root.key' \
-     -O 'https://raw.githubusercontent.com/minio/kes/master/root.cert'
 
 export MINIO_KMS_KES_ENDPOINT=https://play.min.io:7373
-export MINIO_KMS_KES_KEY_FILE=root.key
-export MINIO_KMS_KES_CERT_FILE=root.cert
+export MINIO_KMS_KES_KEY_FILE=tests/config/root.key
+export MINIO_KMS_KES_CERT_FILE=tests/config/root.cert
 export MINIO_KMS_KES_KEY_NAME=my-minio-key
 
 # start the s3 server
diff --git a/tests/br_restore_TDE_enable/workload b/tests/br_restore_TDE_enable/workload
index e3fadf9a3..664fe7ee8 100644
--- a/tests/br_restore_TDE_enable/workload
+++ b/tests/br_restore_TDE_enable/workload
@@ -1,4 +1,4 @@
-recordcount=10000
+recordcount=1000
 operationcount=0
 workload=core
 
diff --git a/tests/config/root.cert b/tests/config/root.cert
new file mode 100644
index 000000000..5f220f79b
--- /dev/null
+++ b/tests/config/root.cert
@@ -0,0 +1,9 @@
+-----BEGIN CERTIFICATE-----
+MIIBKDCB26ADAgECAhB6vebGMUfKnmBKyqoApRSOMAUGAytlcDAbMRkwFwYDVQQD
+DBByb290QHBsYXkubWluLmlvMB4XDTIwMDQzMDE1MjIyNVoXDTI1MDQyOTE1MjIy
+NVowGzEZMBcGA1UEAwwQcm9vdEBwbGF5Lm1pbi5pbzAqMAUGAytlcAMhALzn735W
+fmSH/ghKs+4iPWziZMmWdiWr/sqvqeW+WwSxozUwMzAOBgNVHQ8BAf8EBAMCB4Aw
+EwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAFBgMrZXADQQDZOrGK
+b2ATkDlu2pTcP3LyhSBDpYh7V4TvjRkBTRgjkacCzwFLm+mh+7US8V4dBpIDsJ4u
+uWoF0y6vbLVGIlkG
+-----END CERTIFICATE-----
diff --git a/tests/config/root.key b/tests/config/root.key
new file mode 100644
index 000000000..53a47e25d
--- /dev/null
+++ b/tests/config/root.key
@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEID9E7FSYWrMD+VjhI6q545cYT9YOyFxZb7UnjEepYDRc
+-----END PRIVATE KEY-----

From 4949a6c13c3bd1632958b238db24c64972340c8d Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Mon, 5 Jul 2021 19:10:04 +0800
Subject: [PATCH 06/12] fix

---
 tests/br_restore_TDE_enable/run.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index 6beba499f..081780065 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -16,7 +16,7 @@
 set -eux
 DB="$TEST_NAME"
 TABLE="usertable"
-DB_COUNT=3
+DB_COUNT=1
 
 # start Minio KMS service
 
@@ -81,8 +81,8 @@ for p in $(seq 2); do
       --log-file $BACKUP_LOG \
       --s3.sse AES256
     
-# ensure the tikv data file are encrypted
-bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file --path=/tmp/backup_restore_test/tikv1/db/CURRENT | grep "Aes256Ctr"
+  # ensure the tikv data file are encrypted
+  bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file --path=/tmp/backup_restore_test/tikv1/db/CURRENT | grep "Aes256Ctr"
 
 
   for i in $(seq $DB_COUNT); do
@@ -93,7 +93,6 @@ bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file --path=/t
   echo "restore start..."
   RESTORE_LOG="restore.log"
   rm -f $RESTORE_LOG
-  unset BR_LOG_TO_TERM
   run_br restore full -s "s3://mybucket/$DB?$S3_KEY" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \
       --log-file $RESTORE_LOG 
 

From fcc1c645c7317573983f724a0e22ef972e334a40 Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Mon, 5 Jul 2021 19:20:16 +0800
Subject: [PATCH 07/12] ci test

---
 tests/br_restore_TDE_enable/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index 081780065..693f2cb33 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -76,13 +76,13 @@ for p in $(seq 2); do
   rm -f $BACKUP_LOG
   unset BR_LOG_TO_TERM
 
-  # using --s3.sse AES256 to ensure backup file are encrypted
   run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT$S3_KEY" \
       --log-file $BACKUP_LOG \
       --s3.sse AES256
     
+  ls /tmp/backup_restore_test/tikv1/db/CURRENT 
   # ensure the tikv data file are encrypted
-  bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file --path=/tmp/backup_restore_test/tikv1/db/CURRENT | grep "Aes256Ctr"
+  bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file --path=/tmp/backup_restore_test/tikv1/db/CURRENT
 
 
   for i in $(seq $DB_COUNT); do

From 7566e44e77444eece1ccd682999e59a6d25d17c1 Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Mon, 5 Jul 2021 19:37:25 +0800
Subject: [PATCH 08/12] fix ci

---
 tests/br_restore_TDE_enable/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index 693f2cb33..8bf1f3f10 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -82,7 +82,7 @@ for p in $(seq 2); do
     
   ls /tmp/backup_restore_test/tikv1/db/CURRENT 
   # ensure the tikv data file are encrypted
-  bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file --path=/tmp/backup_restore_test/tikv1/db/CURRENT
+  bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file 
 
 
   for i in $(seq $DB_COUNT); do

From e3ebd0314b4ed4569a5bc3ef67b17d1125e51fd6 Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Mon, 5 Jul 2021 19:48:08 +0800
Subject: [PATCH 09/12] fix ci

---
 tests/br_restore_TDE_enable/run.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index 8bf1f3f10..1570ce70f 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -80,9 +80,8 @@ for p in $(seq 2); do
       --log-file $BACKUP_LOG \
       --s3.sse AES256
     
-  ls /tmp/backup_restore_test/tikv1/db/CURRENT 
   # ensure the tikv data file are encrypted
-  bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file 
+  bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file | grep Aes256Ctr
 
 
   for i in $(seq $DB_COUNT); do

From ab9f3f6c8f0e8baab00553df8939b29f61e6b252 Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Thu, 8 Jul 2021 22:42:39 +0800
Subject: [PATCH 10/12] fix

---
 tests/_utils/run_services | 2 +-
 tests/config/tikv.toml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/_utils/run_services b/tests/_utils/run_services
index fa1f3bb06..41b8f0b08 100644
--- a/tests/_utils/run_services
+++ b/tests/_utils/run_services
@@ -224,7 +224,7 @@ start_services_impl() {
 
     start_pd
     # When using TDE, we add the master key to a file, and this master key is used to encrypt data key
-    echo -e "3b5896b5be691006e0f71c3040a29495ddcad20b14aff61806940ebd780d3c62" > "/tmp/file"
+    echo -e "3b5896b5be691006e0f71c3040a29495ddcad20b14aff61806940ebd780d3c62" > "$TEST_DIR/master-key-file"
     for i in $(seq $TIKV_COUNT); do
         start_tikv "$i"
     done
diff --git a/tests/config/tikv.toml b/tests/config/tikv.toml
index 6e832605f..dc42772a7 100644
--- a/tests/config/tikv.toml
+++ b/tests/config/tikv.toml
@@ -32,4 +32,4 @@ data-encryption-method = "aes256-ctr"
 
 [security.encryption.master-key]
 type = "file"
-path = "/tmp/file"
+path = "/tmp/backup_restore_test/master-key-file"

From ef5e6a42ac934f0d141cafe86412e4f6496f20f0 Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Fri, 9 Jul 2021 00:02:03 +0800
Subject: [PATCH 11/12] work

---
 tests/br_restore_TDE_enable/run.sh | 40 ++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index 1570ce70f..33715ec10 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -16,15 +16,38 @@
 set -eux
 DB="$TEST_NAME"
 TABLE="usertable"
-DB_COUNT=1
+DB_COUNT=3
 
 # start Minio KMS service
+# curl -sSL --tlsv1.2 \
+#     -O 'https://raw.githubusercontent.com/minio/kes/master/root.key' \
+#     -O 'https://raw.githubusercontent.com/minio/kes/master/root.cert'
 
-export MINIO_KMS_KES_ENDPOINT=https://play.min.io:7373
-export MINIO_KMS_KES_KEY_FILE=tests/config/root.key
-export MINIO_KMS_KES_CERT_FILE=tests/config/root.cert
+rm -rf ./keys
+rm -f server.key server.cert
+bin/kes tool identity new --server --key server.key --cert server.cert --ip "127.0.0.1" --dns localhost
+
+
+# create private key and cert for restoration
+rm -f root.key root.cert
+bin/kes tool identity new --key=root.key --cert=root.cert root
+
+bin/kes server --key=server.key --cert=server.cert --root=$(kes tool identity of root.cert) --auth=off &
+KES_pid=$!
+
+sleep 5
+
+export KES_CLIENT_CERT=root.cert
+export KES_CLIENT_KEY=root.key 
+bin/kes key create -k my-minio-key
+
+export MINIO_KMS_KES_ENDPOINT=https://127.0.0.1:7373
+export MINIO_KMS_KES_CERT_FILE=root.cert
+export MINIO_KMS_KES_KEY_FILE=root.key
+export MINIO_KMS_KES_CA_PATH=server.cert
 export MINIO_KMS_KES_KEY_NAME=my-minio-key
 
+
 # start the s3 server
 export MINIO_ACCESS_KEY='KEXI7MANNASOPDLAOIEF'
 export MINIO_SECRET_KEY='MaKYxEGDInMPtEYECXRJLU+FPNKb/wAX/MElir7E'
@@ -53,7 +76,7 @@ start_s3() {
 start_s3
 echo "started s3 with pid = $s3_pid"
 
-bin/mc config --config-dir "$TEST_DIR/$TEST_NAME" \
+bin/mc config --config-dir "$TEST_DIR/$TEST_NAME"  \
     host add minio http://$S3_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
 
 # Fill in the database
@@ -76,12 +99,13 @@ for p in $(seq 2); do
   rm -f $BACKUP_LOG
   unset BR_LOG_TO_TERM
 
+  # using --s3.sse AES256 to ensure backup file are encrypted
   run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT$S3_KEY" \
       --log-file $BACKUP_LOG \
       --s3.sse AES256
     
-  # ensure the tikv data file are encrypted
-  bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file | grep Aes256Ctr
+# ensure the tikv data file are encrypted
+bin/tikv-ctl --config=tests/config/tikv.toml encryption-meta dump-file | grep "Aes256Ctr"
 
 
   for i in $(seq $DB_COUNT); do
@@ -92,6 +116,7 @@ for p in $(seq 2); do
   echo "restore start..."
   RESTORE_LOG="restore.log"
   rm -f $RESTORE_LOG
+  unset BR_LOG_TO_TERM
   run_br restore full -s "s3://mybucket/$DB?$S3_KEY" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \
       --log-file $RESTORE_LOG 
 
@@ -123,3 +148,4 @@ done
 for i in $(seq $DB_COUNT); do
     run_sql "DROP DATABASE $DB${i};"
 done
+kill -9 $KES_pid
\ No newline at end of file

From 3dd36f8fe7e823190c77546d95374d61825b7cf1 Mon Sep 17 00:00:00 2001
From: zwj-coder <2780898229@qq.com>
Date: Tue, 13 Jul 2021 17:34:16 +0800
Subject: [PATCH 12/12] add trap statement

---
 tests/br_restore_TDE_enable/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/br_restore_TDE_enable/run.sh b/tests/br_restore_TDE_enable/run.sh
index 33715ec10..014879237 100755
--- a/tests/br_restore_TDE_enable/run.sh
+++ b/tests/br_restore_TDE_enable/run.sh
@@ -34,6 +34,7 @@ bin/kes tool identity new --key=root.key --cert=root.cert root
 
 bin/kes server --key=server.key --cert=server.cert --root=$(kes tool identity of root.cert) --auth=off &
 KES_pid=$!
+trap 'kill -9 $KES_pid' EXIT
 
 sleep 5
 
@@ -147,5 +148,4 @@ done
 
 for i in $(seq $DB_COUNT); do
     run_sql "DROP DATABASE $DB${i};"
-done
-kill -9 $KES_pid
\ No newline at end of file
+done
\ No newline at end of file