Merge pull request #139 from NethServer/feat-7072

Selective restoration of IMAP folders

Refs NethServer/dev#7072

README.md

@@ -276,7 +276,7 @@ Migration notes:
    its contents by either configuring it as shared, or by creating a
    "root" user in the LDAP database with a new, secure password.
 
-1. Quota temporarly unavailable. The new "quota count" Dovecot backend is
+1. Quota temporarily unavailable. The new "quota count" Dovecot backend is
    used. Large mailboxes need a while to reindex the quota size. During
    reindexing, quota information is not available and the following
    message is logged:

build-images.sh

@@ -39,6 +39,7 @@ buildah add "${container}" ui/dist /ui
 # Setup the entrypoint, ask to reserve one TCP port with the label and set a rootless container
 buildah config --entrypoint=/ \
     --label="org.nethserver.max-per-node=1" \
+    --label="org.nethserver.min-core=3.3.0-0" \
     --label="org.nethserver.images=$(printf "${repobase}/mail-%s:${IMAGETAG:-latest} " \
         dovecot \
         postfix \

dovecot/README.md

@@ -90,6 +90,25 @@ preference is stored in the _dict_ database file
 `/var/lib/dovecot/dict/uspamret.db`. See the dedicated section below to
 modify it.
 
+### `restore-folder`
+
+Import subdirectories of a Maildir++ structure (restored from backup)
+under the user's "Restored folder." Command syntax:
+
+    restore-folder {USER} {BACKUP_DIR} {FOLDER_PATH} [ROOT_FOLDER]
+
+- **USER**: The destination IMAP user name.
+- **BACKUP_DIR**: The source directory where backup contents were
+  extracted. Any subdirectory beginning with a "." (per the Maildir++
+  format) is imported.
+- **FOLDER_PATH**: The IMAP path for the restored folder. Unused path
+  ancestors are pruned.
+- **ROOT_FOLDER** (Optional): Defaults to "Restored folder" if
+  unspecified, under which contents are imported.
+
+The command disables the USER's quota if necessary to avoid an over-quota
+state.
+
 ## Storage
 
 Volumes are:

dovecot/usr/local/bin/restore-folder

@@ -0,0 +1,61 @@
+#!/bin/ash
+# shellcheck shell=bash
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+set -o pipefail
+set -e
+
+user="${1:?missing user argument}"
+restoredir="${2:?missing source dir path}"
+destfolder="${3:?missing destination folder name}"
+rootfolder="${4:-Restored folder}"
+basefolder="$(basename "${destfolder}")"
+rootprefix=$(doveadm mailbox path -u "${user}" "${rootfolder}")
+
+chown -c vmail:vmail "${restoredir}"
+cd "${restoredir}"
+
+read -r curquotalimit curquotamb < <(doveadm -f tab quota get -u "${user}" | awk -F $'\t' 'NR==2 {print $4 "\t" int($4/1024)}')
+if [ "${curquotalimit}" != "-" ]; then
+    # Temporarily disable the user's quota
+    doveadm dict set -u "${user}" fs:posix:prefix=/var/lib/dovecot/dict/uquota/ shared/"${user}" 0
+fi
+
+echo "${rootfolder}" | tr / $'\n' | while read -r rootpart ; do
+    rootacc="${rootacc}${rootacc+/}${rootpart}"
+    doveadm mailbox create    -u "${user}" "${rootacc}" &>/dev/null || :
+    doveadm mailbox subscribe -u "${user}" "${rootacc}"
+done
+doveadm mailbox delete    -u "${user}" -r -s "${rootfolder}/${basefolder}" &>/dev/null || :
+for folder in .* ; do
+    [ "${folder}" == "." ] && continue
+    [ "${folder}" == ".." ] && continue
+    [ ! -d "${folder}" ] && continue
+    if [ -e "${rootprefix}${folder}" ]; then
+        echo "WARNING! Found already existing destination path ${rootprefix}${folder}. Forcing removal." 1>&2
+        rm -rf "${rootprefix}${folder}"
+    fi
+    mv -v "${folder}" "${rootprefix}${folder}"
+done
+# Prune destfolder ancestors:
+if [ "${destfolder}" != "${basefolder}" ]; then
+    doveadm mailbox rename    -u "${user}" -s "${rootfolder}/${destfolder}" "${rootfolder}/${basefolder}" || :
+fi
+doveadm index             -u "${user}" "${rootfolder}/${basefolder}"
+doveadm quota recalc      -u "${user}"
+
+read -r newquotavalue < <(doveadm -f tab quota get -u "${user}" | awk -F $'\t' 'NR==2 {print $3}')
+
+if [ "${curquotalimit}" == "-" ]; then
+    echo "QUOTA_UNCHANGED. Quota usage ${newquotavalue} KB. Quota is not limited, nothing to do."
+elif [ "${newquotavalue}" -lt "${curquotalimit}" ]; then
+    # Restore previous user's quota
+    doveadm dict set -u "${user}" fs:posix:prefix=/var/lib/dovecot/dict/uquota/ shared/"${user}" "${curquotamb}"
+    echo "QUOTA_UNCHANGED. Quota restored to ${curquotamb}"
+else
+    echo "QUOTA_DISABLED_BY_RESTORE. Original quota was insufficient for the restore of ${destfolder}. User quota is disabled now."
+fi

imageroot/actions/list-user-mailboxes/10list_user_mailboxes

@@ -22,7 +22,7 @@ try:
     rquotas = mail.doveadm_query("quotaGet", {"allUsers":True})
 except mail.DoveadmError as ex:
     if ex.code == 75:
-        print(agent.SD_WARNING + "Quota information is temporarly unavailable. Try again later.", ex, file=sys.stderr)
+        print(agent.SD_WARNING + "Quota information is temporarily unavailable. Try again later.", ex, file=sys.stderr)
     else:
         print(agent.SD_ERROR + "quotaGet error", ex, file=sys.stderr)
 except Exception:

imageroot/actions/remove-public-mailbox/20remove_public_mailbox

@@ -27,7 +27,7 @@ try:
         {'alocal':mbxn, 'domain':"*"})
 
     # Destroy the mailbox and its contents
-    mail.doveadm_query("mailboxDelete", {"mailbox": [mbxn], "user": "vmail"})
+    mail.doveadm_query("mailboxDelete", {"mailbox": [mbxn], "user": "vmail", "recursive": True})
 
 except mail.DoveadmError as dex:
     if dex.code != 68:

imageroot/actions/restore-backup-content/50restore_backup_content

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import json
+import sys
+import os
+import subprocess
+import agent
+import atexit
+
+request = json.load(sys.stdin)
+if request['user'] == 'vmail':
+    # Public folders belong to user Vmail. When one of them is restored, the
+    # destination folder is created under the Public folder itself.
+    destroot = request.get("destroot", f"{request['folder']}/Restored folder")
+else:
+    destroot = request.get("destroot", "Restored folder")
+# Temporary path for restore. It is relative to Dovecot's vmail dir, the
+# default working directory for the dovecot container:
+tmp_restore_path = ".restore-backup-content." + os.environ["AGENT_TASK_ID"]
+def cleanup_tmp():
+    global tmp_restore_path
+    if tmp_restore_path:
+        agent.run_helper("podman", "exec", "dovecot", "rm", "-rf", tmp_restore_path)
+atexit.register(cleanup_tmp)
+# Transform the folder path into a filesystem directory name:
+_, maildir_path = subprocess.check_output([
+    'podman', 'exec', 'dovecot',
+    'doveadm', 'mailbox', 'path', '-u', request['user'], request['folder'],
+], text=True).strip().rsplit('/', 1)
+maildir_path_goescaped = maildir_path.replace("\\", "\\\\")
+restic_args = [
+    "restore",
+    "--json",
+    f"{request['snapshot']}:volumes/dovecot-data/{request['user']}/Maildir",
+    f"--target=/srv/volumes/dovecot-data/{tmp_restore_path}",
+    f"--include={maildir_path_goescaped}*",
+]
+podman_args = ["--workdir=/srv"] + agent.agent.get_state_volume_args()
+
+# Prepare progress callback function that captures non-progress messages too:
+last_restic_message = {}
+def build_restore_progress_callback():
+    restore_progress = agent.get_progress_callback(5, 95)
+    def fprog(omessage):
+        global last_restic_message
+        last_restic_message = omessage
+        if omessage['message_type'] == 'status':
+            fpercent = float(omessage['percent_done'])
+            restore_progress(int(fpercent * 100))
+    return fprog
+# Run the restic restore command capturing the progress status:
+prestore = agent.run_restic(agent.redis_connect(), request["destination"], request["repopath"], podman_args, restic_args, progress_callback=build_restore_progress_callback())
+if prestore.returncode != 0:
+    print(agent.SD_ERR + f"Restic restore command failed with exit code {prestore.returncode}.", file=sys.stderr)
+    sys.exit(1)
+
+# 1. Disable the user quota temporarily.
+# 2. Move the restored content under "Restored content" IMAP folder.
+#    Remove the destination if already exists.
+# 3. Reindex the destination folder.
+# 4. Restore the previous quota setting, if possible
+quota_disabled = False
+
+proc_import = subprocess.run(["podman", "exec", "-i", "dovecot",
+    "restore-folder", request["user"], f"/var/lib/vmail/{tmp_restore_path}", request['folder'], destroot],
+    stdout=subprocess.PIPE, text=True)
+if proc_import.returncode != 0:
+    print(agent.SD_ERR + f"Import script failed with exit code {proc_import.returncode}.", file=sys.stderr)
+    sys.exit(1)
+
+if 'QUOTA_DISABLED_BY_RESTORE' in proc_import.stdout:
+    quota_disabled = True
+
+json.dump({
+    "request": request,
+    "quota_disabled": quota_disabled,
+    "last_restic_message": last_restic_message,
+}, fp=sys.stdout)

imageroot/actions/restore-backup-content/validate-input.json

@@ -0,0 +1,52 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "restore-backup-content input",
+    "$id": "http://schema.nethserver.org/mail/restore-backup-content-input.json",
+    "description": "Restore the backup of a user's mail folder.",
+    "examples": [
+        {
+            "folder": "INBOX/Liste/Samba.org",
+            "user": "first.user",
+            "destination": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8",
+            "repopath": "mail/4372a5d5-0886-45d3-82e7-68d913716a4c",
+            "snapshot": "aee022f50fa0b494ee9b172d1395375d7d471e3b647b6238750008dba92d29f7"
+        },
+        {
+            "folder": "INBOX/Liste/Samba.org",
+            "user": "first.user",
+            "destination": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8",
+            "repopath": "mail/4372a5d5-0886-45d3-82e7-68d913716a4c",
+            "snapshot": "latest"
+        }
+    ],
+    "type": "object",
+    "required": [
+        "folder",
+        "destination",
+        "repopath",
+        "snapshot",
+        "user"
+    ],
+    "properties": {
+        "folder": {
+            "type": "string",
+            "description": "Absolute IMAP folder path to restore. Its subfolders will be restored as well."
+        },
+        "destination": {
+            "type": "string",
+            "description": "The UUID of the backup destination where the Restic repository resides."
+        },
+        "repopath": {
+            "type": "string",
+            "description": "Restic repository path, under the backup destination"
+        },
+        "snapshot": {
+            "type": "string",
+            "description": "Restic snapshot ID to restore"
+        },
+        "user": {
+            "type": "string",
+            "description": "Restore the content of this Dovecot IMAP user"
+        }
+    }
+}

imageroot/actions/restore-backup-content/validate-output.json

@@ -0,0 +1,37 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "restore-backup-content output",
+    "$id": "http://schema.nethserver.org/mail/restore-backup-content-output.json",
+    "description": "Restore the backup of a user's mail folder.",
+    "examples": [
+        {
+            "request": {
+                "destination": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8",
+                "folder": "INBOX/Liste/Samba.org",
+                "repopath": "mail/4372a5d5-0886-45d3-82e7-68d913716a4c",
+                "snapshot": "latest",
+                "user": "first.user"
+            },
+            "quota_disabled": false
+        }
+    ],
+    "type": "object",
+    "required": [
+        "quota_disabled"
+    ],
+    "properties": {
+        "quota_disabled": {
+            "title": "Quota disabled",
+            "type": "boolean",
+            "description": "If the restore has disabled the user's quota or not"
+        },
+        "last_restic_message": {
+            "type": "object",
+            "description": "Last JSON message from Restic restore"
+        },
+        "request": {
+            "type": "object",
+            "title": "Original request object"
+        }
+    }
+}

imageroot/actions/seek-snapshot-contents/50seek_snapshot_contents

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import json
+import sys
+import os
+import subprocess
+import agent
+
+request = json.load(sys.stdin)
+rdb = agent.redis_connect()
+podman_args = ["--workdir=/srv"] + agent.agent.get_state_volume_args()
+restic_args = [
+    "dump",
+    request["snapshot"],
+    "/state/maildir-dump.lst",
+]
+
+restic_cmd, restic_env = agent.prepare_restic_command(rdb, request["destination"], request["repopath"], podman_args, restic_args) 
+with subprocess.Popen(restic_cmd, stdout=subprocess.PIPE, stderr=sys.stderr, text=True, env=restic_env) as vproc:
+    folders = []
+    while True:
+        line = vproc.stdout.readline()
+        if not line:
+            break
+        fields = line.rstrip().split("\t")
+        if fields[0] == request["user"]:
+            folders.append(fields[1])
+if vproc.returncode != 0:
+    print(agent.SD_ERR + f"Restic dump command failed with exit code {vproc.returncode}.", file=sys.stderr)
+    sys.exit(1)
+
+folders.sort()
+json.dump({
+    "request": request,
+    "folders": folders,
+}, fp=sys.stdout)

imageroot/actions/seek-snapshot-contents/validate-input.json

@@ -0,0 +1,45 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "seek-snapshot-contents input",
+    "$id": "http://schema.nethserver.org/mail/seek-snapshot-contents-input.json",
+    "description": "Extract the list of IMAP folders from a backup snapshot",
+    "examples": [
+        {
+            "destination": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8",
+            "user": "first.user",
+            "repopath": "mail/4372a5d5-0886-45d3-82e7-68d913716a4c",
+            "snapshot": "aee022f50fa0b494ee9b172d1395375d7d471e3b647b6238750008dba92d29f7"
+        },
+        {
+            "destination": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8",
+            "user": "first.user",
+            "repopath": "mail/4372a5d5-0886-45d3-82e7-68d913716a4c",
+            "snapshot": "latest"
+        }
+    ],
+    "type": "object",
+    "required": [
+        "destination",
+        "repopath",
+        "snapshot",
+        "user"
+    ],
+    "properties": {
+        "destination": {
+            "type": "string",
+            "description": "The UUID of the backup destination where the Restic repository resides."
+        },
+        "repopath": {
+            "type": "string",
+            "description": "Restic repository path, under the backup destination"
+        },
+        "snapshot": {
+            "type": "string",
+            "description": "Restic snapshot ID to restore"
+        },
+        "user": {
+            "type": "string",
+            "description": "Restore the content of this Dovecot IMAP user"
+        }
+    }
+}