Implement overlay deref command

copy changes from upperdir to a new upperdir unfolding redirect and metacopy
to get a portable overlay upperdir layer.

Signed-off-by: Amir Goldstein <[email protected]>

README.md

@@ -7,6 +7,7 @@ This program comes provides three tools:
 - **vacuum** - remove duplicated files in `upperdir` where `copy_up` is done but the file is not actually modified (see the sentence "the `copy_up` may turn out to be unnecessary" in the [Linux documentation](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt)). This may reduce the size of `upperdir` without changing `lowerdir` or `overlay`.
 - **diff** - show the list of actually changed files (the difference between `overlay` and `lowerdir`). A file with its type changed (i.e. from symbolic link to regular file) will shown as deleted then added, rather than modified. Similarly, for a opaque directory in `upperdir`, the corresponding directory in `lowerdir` (if exists) will be shown as entirely deleted, and a new directory with the same name added. File permission/owner changes will be simply shown as modified.
 - **merge** - merge down the changes from `upperdir` to `lowerdir`. Unlike [aubrsync](http://aufs.sourceforge.net/aufs2/brsync/README.txt) for AuFS which bypasses the union filesystem mechanism, overlayfs-utils emulates the OverlayFS logic, which will be far more efficient. After this operation, `upperdir` will be empty and `lowerdir` will be the same as original `overlay`.
+- **deref** - copy changes from `upperdir` to `uppernew` while unfolding redirect directories and metacopy regular files, so that new upperdir is compatible with legacy overlayfs driver.
 
 For safety reasons, vacuum and merge will not actually modify the filesystem, but generate a shell script to do the changes instead.
 

logic.c

@@ -550,6 +550,21 @@ int traverse(const char *lower_root, const char *upper_root, FILE* script_stream
     return fts_close(ftsp) || return_val;
 }
 
+static int deref_d(const char *mnt_path, const char* upper_path, const size_t mnt_root_len, const struct stat *mnt_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool redirect;
+    if (is_redirect(upper_path, &redirect) < 0) { return -1; }
+    if (!redirect) { return 0; }
+    *fts_instr = FTS_SKIP;
+    return command(script_stream, "rm -rf %U", upper_path) || command(script_stream, "cp -a %M %U", mnt_path, upper_path);
+}
+
+static int deref_f(const char *mnt_path, const char* upper_path, const size_t mnt_root_len, const struct stat *mnt_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) {
+    bool metacopy;
+    if (is_metacopy(upper_path, &metacopy) < 0) { return -1; }
+    if (!metacopy) { return 0; }
+    return command(script_stream, "rm -r %U", upper_path) || command(script_stream, "cp -a %M %U", mnt_path, upper_path);
+}
+
 int vacuum(const char* lowerdir, const char* upperdir, FILE* script_stream) {
     return traverse(lowerdir, upperdir, script_stream, vacuum_d, vacuum_dp, vacuum_f, vacuum_sl, NULL);
 }
@@ -561,3 +576,7 @@ int diff(const char* lowerdir, const char* upperdir) {
 int merge(const char* lowerdir, const char* upperdir, FILE* script_stream) {
     return traverse(lowerdir, upperdir, script_stream, merge_d, merge_dp, merge_f, merge_sl, merge_whiteout);
 }
+
+int deref(const char* mountdir, const char* upperdir, FILE* script_stream) {
+    return traverse(mountdir, upperdir, script_stream, deref_d, NULL, deref_f, NULL, NULL);
+}

logic.h

@@ -27,4 +27,11 @@ int diff(const char* lowerdir, const char* upperdir);
  */
 int merge(const char* lowerdir, const char* upperdir, FILE* script_stream);
 
+/*
+ * Unfold metacopy and redirect upper.
+ *
+ * mountdir is required and lowerdir is irrelevant.
+ */
+int deref(const char* mountdir, const char* upperdir, FILE* script_stream);
+
 #endif //OVERLAYFS_TOOLS_LOGIC_H

main.c

@@ -28,10 +28,12 @@ void print_help() {
     puts("  vacuum - remove duplicated files in upperdir where copy_up is done but the file is not actually modified");
     puts("  diff   - show the list of actually changed files");
     puts("  merge  - merge all changes from upperdir to lowerdir, and clear upperdir");
+    puts("  deref  - copy changes from upperdir to a new upperdir unfolding redirect and metacopy");
     puts("");
     puts("Options:");
     puts("  -l, --lowerdir=LOWERDIR    the lowerdir of OverlayFS (required)");
     puts("  -u, --upperdir=UPPERDIR    the upperdir of OverlayFS (required)");
+    puts("  -m, --mountdir=MOUNTDIR    the mountdir of OverlayFS (optional)");
     puts("  -L, --lowernew=LOWERNEW    the lowerdir of new OverlayFS (optional)");
     puts("  -U, --uppernew=UPPERNEW    the upperdir of new OverlayFS (optional)");
     puts("  -v, --verbose              with diff action only: when a directory only exists in one version, still list every file of the directory");
@@ -127,11 +129,12 @@ int main(int argc, char *argv[]) {
 
     char *lower = NULL;
     char *upper = NULL;
-    char *dir;
+    char *dir, *mnt = NULL;
 
     static struct option long_options[] = {
         { "lowerdir", required_argument, 0, 'l' },
         { "upperdir", required_argument, 0, 'u' },
+        { "mountdir", required_argument, 0, 'm' },
         { "lowernew", required_argument, 0, 'L' },
         { "uppernew", required_argument, 0, 'U' },
         { "help",     no_argument      , 0, 'h' },
@@ -142,7 +145,7 @@ int main(int argc, char *argv[]) {
 
     int opt = 0;
     int long_index = 0;
-    while ((opt = getopt_long_only(argc, argv, "l:u:L:U:hvb", long_options, &long_index)) != -1) {
+    while ((opt = getopt_long_only(argc, argv, "l:u:m:L:U:hvb", long_options, &long_index)) != -1) {
         switch (opt) {
             case 'l':
                 lower = realpath(optarg, NULL);
@@ -152,6 +155,10 @@ int main(int argc, char *argv[]) {
                 upper = realpath(optarg, NULL);
                 if (upper) { vars[UPPERDIR] = upper; }
                 break;
+            case 'm':
+                mnt = realpath(optarg, NULL);
+                if (mnt) { vars[MOUNTDIR] = mnt; }
+                break;
             case 'L':
                 directory_create("New lowerdir", optarg);
                 dir = realpath(optarg, NULL);
@@ -218,6 +225,15 @@ int main(int argc, char *argv[]) {
             script = create_shell_script(filename_template);
             if (script == NULL) { fprintf(stderr, "Script file cannot be created.\n"); return EXIT_FAILURE; }
             out = merge(lower, upper, script);
+        } else if (strcmp(argv[optind], "deref") == 0) {
+            if (!mnt || !vars[UPPERNEW]) { fprintf(stderr, "'deref' command requires --uppernew and --mountdir.\n"); return EXIT_FAILURE; }
+            if (!directory_exists(mnt)) {
+                fprintf(stderr, "OverlayFS mount directory cannot be opened.\n");
+                goto see_help;
+            }
+            script = create_shell_script(filename_template);
+            if (script == NULL) { fprintf(stderr, "Script file cannot be created.\n"); return EXIT_FAILURE; }
+            out = deref(mnt, upper, script);
         } else {
             fprintf(stderr, "Action not supported.\n");
             goto see_help;

sh.c

@@ -11,6 +11,7 @@ char * vars[NUM_VARS];
 const char * var_names[NUM_VARS] = {
     "LOWERDIR",
     "UPPERDIR",
+    "MOUNTDIR",
     "LOWERNEW",
     "UPPERNEW",
 };

sh.h

@@ -4,6 +4,7 @@
 enum {
     LOWERDIR,
     UPPERDIR,
+    MOUNTDIR,
     LOWERNEW,
     UPPERNEW,
     NUM_VARS