fix: virtual host url

README.md

@@ -26,7 +26,7 @@
 # 使用文件备份
 MONGO_URI=mongodb://username:[email protected] BACKUP_PATH=./backup FILE_PREFIX=tmp python docker/backup.py
 # 启用 s3 备份
-MONGO_URI=mongodb://username:[email protected] BACKUP_PATH=./backup FILE_PREFIX=tmp S3_ENABLE=1 S3_EP="https://minio-api.36node.com" S3_ACCESS_KEY="xxxx" S3_ACCESS_SECRET="xxxx" S3_BUCKET="test" S3_PREFIX="prefix" python docker/backup.py
+MONGO_URI=mongodb://username:[email protected] BACKUP_PATH=./backup FILE_PREFIX=tmp S3_ENABLE=true S3_EP="https://minio-api.36node.com" S3_ACCESS_KEY="xxxx" S3_ACCESS_SECRET="xxxx" S3_BUCKET="test" S3_PREFIX="prefix" python docker/backup.py
 
 # 使用文件恢复
 MONGO_URI=mongodb://username:[email protected] BACKUP_PATH=./backup python docker/restore.py
@@ -116,14 +116,21 @@ kubectl -n mongodb-backup get pod
 kubectl -n mongodb-backup exec -it restore-xxx-xxx -- python3 /app/restore.py
 ```
 
-#### 存储
+### 存储
 
 支持 挂载磁盘 或 PVC，容器内的挂载路径默认为 `/backup`
 
 - 本地磁盘，指定 nodeSelector 及 hostPath
 - PVC，指定 existingClaim
 - comming feature 支持 storage_class
 
+### 关于 S3 endpoint 的说明
+
+S3 支持使用虚拟域名作为endpoint，即可以将 region 或者 bucket 放入域名中使用，不同的 S3 配置会略有不同，需进行测试。其他配置，可参考[boto3文档](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html)
+
+- 36node 自建 minio，使用默认配置即可
+- xxx.aliyuncs.com，需要设置 S3_EP_VIRTUAL 为 true
+
 ## 环境变量说明
 
 ### backup
@@ -135,16 +142,18 @@ kubectl -n mongodb-backup exec -it restore-xxx-xxx -- python3 /app/restore.py
 - MONGO_EXCLUDE_COLLECTIONS: 选填，忽略的集合名称，支持多个，例如 test1,test2，若不为空，则需保证 MONGO_DB 也存在，且若 MONGO_COLLECTION 不为空，则忽略该参数
 - BACKUP_PWD: 选填，加密密码，备份文件可用 zip 加密
 
-- S3_ENABLE: 选填，是否启用 S3 存储备份，不为空值即认为启用，例如 1 视为启用 S3
+- S3_ENABLE: 选填，是否启用 S3 存储备份，true 表示启用
 - S3_EP: 选填，S3 url，例如 https://minio-api.36node.com
+- S3_EP_VIRTUAL: 选填，是否启用虚拟 host url，true 表示启用
 - S3_ACCESS_KEY: 选填，S3 access key
 - S3_ACCESS_SECRET: 选填，S3 access secret
+- S3_REGION: 选填，地区名
 - S3_BUCKET: 选填，要存储的桶名
 - S3_PREFIX: 选填，要存储的前缀
 
 ### restore
 
-- RESTORE_FROM_S3: 选填，是否从 S3 中进行恢复，不为空值即认为启用，例如 1 视为从 S3 中进行恢复
+- S3_ENABLE: 选填，是否从 S3 中进行恢复，true 表示启用
 
 同 backup 的变量
 
@@ -154,8 +163,10 @@ kubectl -n mongodb-backup exec -it restore-xxx-xxx -- python3 /app/restore.py
 - BACKUP_PWD
 
 - S3_EP
+- S3_EP_VIRTUAL
 - S3_ACCESS_KEY
 - S3_ACCESS_SECRET
+- S3_REGION
 - S3_BUCKET
 - S3_PREFIX
 

docker/backup.py

@@ -5,6 +5,7 @@
 import subprocess
 from urllib.parse import urlparse
 import boto3
+from botocore.client import Config
 
 # 数据库清理备份脚本
 # 1. 按要求备份数据，并保存到指定路径
@@ -25,11 +26,13 @@
     "MONGO_COLLECTION",
     "MONGO_EXCLUDE_COLLECTIONS",
     "BACKUP_PWD",
-
+    # S3 CONFIG
     "S3_ENABLE",
     "S3_EP",
+    "S3_EP_VIRTUAL",
     "S3_ACCESS_KEY",
     "S3_ACCESS_SECRET",
+    "S3_REGION",
     "S3_BUCKET",
     "S3_PREFIX",
 ]
@@ -46,25 +49,45 @@ def check_var(key):
     return False
 
 
+def check_bool(key):
+    if os.environ[key].lower() == "true":
+        return True
+    return False
+
+
 # 必填
 uri = os.environ["MONGO_URI"]
 
 # 选填
-file_prefix = os.environ["FILE_PREFIX"] if check_var("FILE_PREFIX") else DEFAULT_FILE_PREFIX
-backup_path = os.environ["BACKUP_PATH"] if check_var("BACKUP_PATH") else DEFAULT_BACKUP_PATH
-backup_save_nums = int(os.environ["BACKUP_SAVE_NUMS"]) if check_var("BACKUP_SAVE_NUMS") else DEFAULT_BACKUP_SAVE_NUMS
+file_prefix = (
+    os.environ["FILE_PREFIX"] if check_var("FILE_PREFIX") else DEFAULT_FILE_PREFIX
+)
+backup_path = (
+    os.environ["BACKUP_PATH"] if check_var("BACKUP_PATH") else DEFAULT_BACKUP_PATH
+)
+backup_save_nums = (
+    int(os.environ["BACKUP_SAVE_NUMS"])
+    if check_var("BACKUP_SAVE_NUMS")
+    else DEFAULT_BACKUP_SAVE_NUMS
+)
 collection = os.environ["MONGO_COLLECTION"] if check_var("MONGO_COLLECTION") else None
 excludeCollections = (
-    os.environ["MONGO_EXCLUDE_COLLECTIONS"].split(",") if check_var("MONGO_EXCLUDE_COLLECTIONS") else None
+    os.environ["MONGO_EXCLUDE_COLLECTIONS"].split(",")
+    if check_var("MONGO_EXCLUDE_COLLECTIONS")
+    else None
 )
 backup_pwd = os.environ["BACKUP_PWD"] if check_var("BACKUP_PWD") else None
 
-s3_enable = True if check_var("S3_ENABLE") else False
+s3_enable = check_bool("S3_ENABLE") if check_var("S3_ENABLE") else False
 s3_ep = os.environ["S3_EP"] if check_var("S3_EP") else None
+s3_ep_virtual = check_bool("S3_EP_VIRTUAL") if check_var("S3_EP_VIRTUAL") else False
 s3_access_key = os.environ["S3_ACCESS_KEY"] if check_var("S3_ACCESS_KEY") else None
-s3_access_secret = os.environ["S3_ACCESS_SECRET"] if check_var("S3_ACCESS_SECRET") else None
+s3_access_secret = (
+    os.environ["S3_ACCESS_SECRET"] if check_var("S3_ACCESS_SECRET") else None
+)
 s3_bucket = os.environ["S3_BUCKET"] if check_var("S3_BUCKET") else None
 s3_prefix = os.environ["S3_PREFIX"] if check_var("S3_PREFIX") else DEFAULT_S3_PREFIX
+s3_region = os.environ["S3_REGION"] if check_var("S3_REGION") else None
 
 # 计算当前日期，按照 年月日时分 格式
 date = (datetime.utcnow() + timedelta(hours=8)).strftime("%Y%m%d%H%M%S")
@@ -149,30 +172,46 @@ def backup_file(prefix):
         subprocess.call(f"zip -e {source}.crypt -P {backup_pwd} {source}", shell=True)
         os.remove(source)
 
+
 def upload_s3(prefix):
+    # config
+    config_s3 = {}
+    if s3_ep_virtual:
+        config_s3["addressing_style"] = "virtual"
+
+    if s3_region:
+        config = Config(s3=config_s3, region_name=s3_region)
+    else:
+        config = Config(s3=config_s3)
+
     client = boto3.client(
         "s3",
         endpoint_url=s3_ep,
         aws_access_key_id=s3_access_key,
         aws_secret_access_key=s3_access_secret,
+        config=config,
     )
 
     # 上传备份文件
-    file_name = f"{prefix}{date}.tar.gz.crypt" if backup_pwd else f"{prefix}{date}.tar.gz"
+    file_name = (
+        f"{prefix}{date}.tar.gz.crypt" if backup_pwd else f"{prefix}{date}.tar.gz"
+    )
     upload_path = f"{s3_prefix}/{file_name}" if s3_prefix else file_name
     client.upload_file(f"{backup_path}/{file_name}", s3_bucket, upload_path)
 
     # 清理 S3 上的多余备份文件
     list_prefix = f"{s3_prefix}/" if s3_prefix else ""
-    resp = client.list_objects_v2(Bucket=s3_bucket, Prefix=list_prefix, Delimiter='/')
+    resp = client.list_objects_v2(Bucket=s3_bucket, Prefix=list_prefix, Delimiter="/")
     if "Contents" in resp:
         objects = resp["Contents"]
 
         file_prefix = f"{list_prefix}{prefix}"
         # 构造正则表达式
         regex_pattern = f"^{re.escape(file_prefix)}(\\d{{14}})\\.tar\\.gz(\\.crypt)?$"
         compiled_regex = re.compile(regex_pattern)
-        keys = [object["Key"] for object in objects if compiled_regex.match(object["Key"])]
+        keys = [
+            object["Key"] for object in objects if compiled_regex.match(object["Key"])
+        ]
 
         # 根据文件名排序（这使得最新的备份文件位于列表的末尾）
         keys.sort()
@@ -182,9 +221,13 @@ def upload_s3(prefix):
 
         # 删除旧的备份文件
         if keys_to_remove:
-            client.delete_objects(Bucket=s3_bucket, Delete={"Objects": [{"Key": key} for key in keys_to_remove]})
+            client.delete_objects(
+                Bucket=s3_bucket,
+                Delete={"Objects": [{"Key": key} for key in keys_to_remove]},
+            )
             print(f"Deleted s3 old backup files: {keys_to_remove}")
 
+
 try:
     if not os.path.exists(backup_path):
         os.makedirs(backup_path)

docker/restore.py

@@ -4,6 +4,7 @@
 from datetime import datetime, timedelta
 import subprocess
 import boto3
+from botocore.client import Config
 
 # 数据库还原
 
@@ -16,13 +17,15 @@
     "FILE_PREFIX",
     "BACKUP_PATH",
     "BACKUP_PWD",
-
-    "RESTORE_FROM_S3",
+    # S3 CONFIG
+    "S3_ENABLE",
     "S3_EP",
+    "S3_EP_VIRTUAL",
     "S3_ACCESS_KEY",
     "S3_ACCESS_SECRET",
     "S3_BUCKET",
     "S3_PREFIX",
+    "S3_REGION",
 ]
 
 for key in must_inputs:
@@ -37,19 +40,31 @@ def check_var(key):
     return False
 
 
+def check_bool(key):
+    if os.environ[key].lower() == "true":
+        return True
+    return False
+
+
 # 必填
 uri = os.environ["MONGO_URI"]
 
 # 选填
-backup_path = os.environ["BACKUP_PATH"] if check_var("BACKUP_PATH") else DEFAULT_BACKUP_PATH
+backup_path = (
+    os.environ["BACKUP_PATH"] if check_var("BACKUP_PATH") else DEFAULT_BACKUP_PATH
+)
 backup_pwd = os.environ["BACKUP_PWD"] if check_var("BACKUP_PWD") else None
 
-restore_from_s3 = True if check_var("RESTORE_FROM_S3") else False
+s3_enable = check_bool("S3_ENABLE") if check_var("S3_ENABLE") else False
 s3_ep = os.environ["S3_EP"] if check_var("S3_EP") else None
+s3_ep_virtual = check_bool("S3_EP_VIRTUAL") if check_var("S3_EP_VIRTUAL") else False
 s3_access_key = os.environ["S3_ACCESS_KEY"] if check_var("S3_ACCESS_KEY") else None
-s3_access_secret = os.environ["S3_ACCESS_SECRET"] if check_var("S3_ACCESS_SECRET") else None
+s3_access_secret = (
+    os.environ["S3_ACCESS_SECRET"] if check_var("S3_ACCESS_SECRET") else None
+)
 s3_bucket = os.environ["S3_BUCKET"] if check_var("S3_BUCKET") else None
 s3_prefix = os.environ["S3_PREFIX"] if check_var("S3_PREFIX") else DEFAULT_S3_PREFIX
+s3_region = os.environ["S3_REGION"] if check_var("S3_REGION") else None
 
 date = (datetime.utcnow() + timedelta(hours=8)).strftime("%Y%m%d%H%M%S")
 
@@ -65,26 +80,29 @@ def get_files(compiled_regex):
     matched_files.sort(reverse=True)
     return matched_files
 
+
 def get_keys_from_s3(client, compiled_regex):
     list_prefix = f"{s3_prefix}/" if s3_prefix else ""
-    resp = client.list_objects_v2(Bucket=s3_bucket, Prefix=list_prefix, Delimiter='/')
+    resp = client.list_objects_v2(Bucket=s3_bucket, Prefix=list_prefix, Delimiter="/")
     if "Contents" in resp:
         objects = resp["Contents"]
 
-        keys = [object["Key"] for object in objects if compiled_regex.match(object["Key"])]
+        keys = [
+            object["Key"] for object in objects if compiled_regex.match(object["Key"])
+        ]
 
         if keys:
             # 根据文件名排序（这使得最新的备份文件位于列表的末尾）
             keys.sort(reverse=True)
             return keys
     raise Exception("没有可用的备份")
 
+
 def download_file(client, key):
     file_name = os.path.basename(key)
     save_path = f"/tmp/{file_name}"
     client.download_file(s3_bucket, key, save_path)
     return save_path
-
 
 
 def restore_file(file_path):
@@ -99,7 +117,9 @@ def restore_file(file_path):
         # 先解密
         unzip_path = os.path.dirname(file_path)
         # -o 覆盖已有文件，-j 不保留文件夹
-        subprocess.call(f"unzip -P {backup_pwd} -oj {file_path} -d {unzip_path}", shell=True)
+        subprocess.call(
+            f"unzip -P {backup_pwd} -oj {file_path} -d {unzip_path}", shell=True
+        )
 
     # 恢复数据
     cmd = f'mongorestore --uri="{uri}" --gzip --archive={restore_path}'
@@ -117,31 +137,42 @@ def restore_file(file_path):
 
     client = None
     # 1. 获取备份文件列表
-    if restore_from_s3:
+    if s3_enable:
+        # s3 config
+        config_s3 = {}
+        if s3_ep_virtual:
+            config_s3["addressing_style"] = "virtual"
+
+        if s3_region:
+            config = Config(s3=config_s3, region_name=s3_region)
+        else:
+            config = Config(s3=config_s3)
+
         client = boto3.client(
             "s3",
             endpoint_url=s3_ep,
             aws_access_key_id=s3_access_key,
             aws_secret_access_key=s3_access_secret,
+            config=config,
         )
         backup_files = get_keys_from_s3(client, compiled_regex)
     else:
-      backup_files = get_files(compiled_regex)
+        backup_files = get_files(compiled_regex)
 
     input_cmd = "请选择要还原的备份文件：\n"
     for index, item in enumerate(backup_files):
         input_cmd += f"{index + 1}. {item}\n"
     value = input(input_cmd)
 
-    if restore_from_s3:
+    if s3_enable:
         # 需要下载文件
         file = download_file(client, backup_files[int(value) - 1])
     else:
         file = f"{backup_path}/{backup_files[int(value) - 1]}"
 
     restore_file(file)
 
-    if restore_from_s3:
+    if s3_enable:
         # 删除临时文件
         os.remove(file)
 

helm-chart/templates/backup.yaml

@@ -35,14 +35,19 @@ spec:
                   value: "true"
                 - name: "S3_EP"
                   value: {{ index .Values.storage.s3 "endpoint" | quote }}
+                - name: "S3_EP_VIRTUAL"
+                  value: {{ index .Values.storage.s3 "endpoint_virtual" | quote }}
                 - name: "S3_ACCESS_KEY"
                   value: {{ index .Values.storage.s3 "access_key" | quote }}
                 - name: "S3_ACCESS_SECRET"
                   value: {{ index .Values.storage.s3 "access_secret" | quote }}
+                - name: "S3_REGION"
+                  value: {{ index .Values.storage.s3 "region" | quote }}
                 - name: "S3_BUCKET"
                   value: {{ index .Values.storage.s3 "bucket" | quote }}
                 - name: "S3_PREFIX"
                   value: {{ index .Values.storage.s3 "prefix" | quote }}
+
                 {{- end }}
                 {{- range $key, $value := .Values.backup.env }}
                 - name: "{{ $key }}"

helm-chart/templates/restore.yaml

@@ -35,10 +35,14 @@ spec:
               value: "true"
             - name: "S3_EP"
               value: {{ index .Values.storage.s3 "endpoint" | quote }}
+            - name: "S3_EP_VIRTUAL"
+              value: {{ index .Values.storage.s3 "endpoint_virtual" | quote }}
             - name: "S3_ACCESS_KEY"
               value: {{ index .Values.storage.s3 "access_key" | quote }}
             - name: "S3_ACCESS_SECRET"
               value: {{ index .Values.storage.s3 "access_secret" | quote }}
+            - name: "S3_REGION"
+              value: {{ index .Values.storage.s3 "region" | quote }}
             - name: "S3_BUCKET"
               value: {{ index .Values.storage.s3 "bucket" | quote }}
             - name: "S3_PREFIX"