Merge branch 'alt-mains-tidy-up' into sort-on-path

# Conflicts:
#	CHANGELOG.md

CHANGELOG.md

@@ -5,9 +5,10 @@ All notable changes to this project will be documented in this file.
 The format follows [keepachangelog.com]. Please stick to it.
 
 ## [2.11.0 Evolving Echidna -- unreleased]
-
 ### Added
 
+* Implement --hash-uniques option to generate full checksums of unique files too.
+* Implement --hash-unmatched option, similar to --hash-uniques but only for size twins.
 * Implement --rank-by f option to rank originals by directory full path
 
 ## [2.10.1 Ludicrous Lemur] -- 2020-06-13

SConstruct

@@ -515,7 +515,7 @@ AddOption(
     action='store', metavar='DIR', help='libdir name (lib or lib64)'
 )
 
-for suffix in ['libelf', 'gettext', 'fiemap', 'blkid', 'json-glib', 'gui']:
+for suffix in ['libelf', 'gettext', 'fiemap', 'blkid', 'gui']:
     AddOption(
         '--without-' + suffix, action='store_const', default=False, const=False,
         dest='with_' + suffix
@@ -599,18 +599,12 @@ conf.env['HAVE_BLKID'] = 0
 conf.check_pkg('blkid', 'HAVE_BLKID', required=False)
 
 conf.env['HAVE_JSON_GLIB'] = 0
-conf.check_pkg('json-glib-1.0', 'HAVE_JSON_GLIB', required=False)
+conf.check_pkg('json-glib-1.0', 'HAVE_JSON_GLIB', required=True)
 
-if GetOption('with_json-glib') is False:
-    conf.env['HAVE_JSON_GLIB'] = 0
-
-packages = ['glib-2.0']
+packages = ['glib-2.0', 'json-glib-1.0']
 if conf.env['HAVE_BLKID']:
     packages.append('blkid')
 
-if conf.env['HAVE_JSON_GLIB']:
-    packages.append('json-glib-1.0')
-
 if conf.env['HAVE_GIO_UNIX']:
     packages.append('gio-unix-2.0')
 
@@ -885,7 +879,6 @@ if 'config' in COMMAND_LINE_TARGETS:
     Support for SHA512 (needs glib >= 2.31)               : {sha512}
     Build manpage from docs/rmlint.1.rst                  : {sphinx}
     Support for caching checksums in file's xattr         : {xattr}
-    Support for reading json caches (needs json-glib)     : {json_glib}
     Checking for proper support of big files >= 4GB       : {bigfiles}
         (needs either sizeof(off_t) >= 8 ...)             : {bigofft}
         (... or presence of stat64)                       : {bigstat}
@@ -916,7 +909,6 @@ Type 'scons' to actually compile rmlint now. Good luck.
             locale=yesno(env['HAVE_LIBINTL']),
             msgfmt=yesno(env['HAVE_MSGFMT']),
             xattr=yesno(env['HAVE_XATTR']),
-            json_glib=yesno(env['HAVE_JSON_GLIB']),
             nonrotational=yesno(env['HAVE_GIO_UNIX'] & env['HAVE_BLKID']),
             gio_unix=yesno(env['HAVE_GIO_UNIX']),
             blkid=yesno(env['HAVE_BLKID']),

docs/developers.rst

@@ -198,11 +198,6 @@ Arguments
     Do not link with ``libblkid``, which is needed to differentiate between
     normal rotational harddisks and non-rotational disks.
 
-:--without-json-glib:
-
-    Do not link with ``libjson-glib``, which is needed to load json-cache files.
-    Without this library a warning is printed when using ``--replay``.
-
 :--without-fiemap:
 
     Do not attempt to use the ``FIEMAP ioctl(2)``. 

docs/install.rst

@@ -16,13 +16,13 @@ Hard dependencies:
 ~~~~~~~~~~~~~~~~~~
 
 * **glib** :math:`\geq 2.32` (general C Utility Library)
+* **libjson-glib** (parsing rmlint's own json as caching layer)
 
 Soft dependencies:
 ~~~~~~~~~~~~~~~~~~
 
 * **libblkid** (detecting mountpoints)
 * **libelf** (nonstripped binary detection)
-* **libjson-glib** (parsing rmlint's own json as caching layer)
 
 Build dependencies:
 ~~~~~~~~~~~~~~~~~~~

docs/rmlint.1.rst

@@ -799,7 +799,7 @@ OTHER STAND-ALONE COMMANDS
 
     By default this will use hashing to compare the files and/or directories.
 
-:``rmlint --dedupe [-r] [-v|-V] <src> <dest>``:
+:``rmlint --dedupe [OPTION…] <src> <dest>``:
 
     If the filesystem supports files sharing physical storage between multiple
     files, and if ``src`` and ``dest`` have same content, this command makes the
@@ -808,26 +808,33 @@ OTHER STAND-ALONE COMMANDS
 
     This command is similar to ``cp --reflink=always <src> <dest>``
     except that it (a) checks that ``src`` and ``dest`` have identical data, and
-    it makes no changes to ``dest``'s metadata.
+    (b) it makes no changes to ``dest``'s metadata.
+
+    Options:
+    * -h, --help               Show help options
+    * -x, --xattr              Check extended attributes to see if the file is already deduplicated
+    * -r, --readonly           Even dedupe read-only [btrfs] snapshots (needs root)
+    * -f, --followlinks        Follow symlinks
+    * -i, --inline-extents     Try to dedupe files with inline extents
+    * -v, --loud               Be more verbose (-vvv for much more)
+    * -V, --quiet              Be less verbose (-VVV for much less)
 
-    Running with ``-r`` option will enable deduplication of read-only [btrfs]
-    snapshots (requires root).
 
 :``rmlint --is-reflink [-v|-V] <file1> <file2>``:
     Tests whether ``file1`` and ``file2`` are reflinks (reference same data).
     This command makes ``rmlint`` exit with one of the following exit codes:
 
-    * 0: files are reflinks
-    * 1: files are not reflinks
-    * 3: not a regular file
-    * 4: file sizes differ
-    * 5: fiemaps can't be read
-    * 6: file1 and file2 are the same path
-    * 7: file1 and file2 are the same file under different mountpoints
-    * 8: files are hardlinks
-    * 9: files are symlinks
-    * 10: files are not on same device
-    * 11: other error encountered
+    * 0: Files are reflinks
+    * 1: An error occurred during checking
+    * 3: Not a regular file
+    * 4: File sizes differ
+    * 5: Files have inline extents
+    * 6: Same file and path
+    * 7: Same file but with different path
+    * 8: Hardlink
+    * 9: Symlink
+    * 10: Files are on different devices
+    * 11: Not linked
 
 
 EXAMPLES

docs/tutorial.rst

@@ -797,6 +797,21 @@ Here's just a list of options that are nice to know, but are not essential:
   processed file. But be sure to read the caveats stated in the `manpage`_!
   Especially keep in mind that you need to have write access to the files for this to work.
 
+  Note that the above example will only save checksums for duplicate files.
+  Alternatively, with the ``--hash-unmatched`` options, checksums are calulated
+  and saved in xattributes for all files that have "size twins", ie files of
+  the same length.  This may make the first run very slow but will greatly speed up
+  future runs.
+
+  .. code-block:: python
+
+    $ rmlint large_dataset/ --xattr --hash-unmatched
+    $ rmlint large_dataset/ --xattr
+
+  There is also the the ``--hash-uniques`` option, which is similar to
+  ``--hash-unmatched`` but also hashes files with no size twins.  This will be
+  even slower than ``--hash-unmatched`` on the first run.
+
 - ``-r`` (``--hidden``): Include hidden files and directories.  The default
   is to ignore these, to save you from destroying git repositories (or similar
   programs) that save their information in a ``.git`` directory where ``rmlint``
@@ -877,6 +892,8 @@ little `tool`_ which makes it really easy to extract data from a ``json`` file:
     $ rmlint t -o json -o uniques:unique_files |  jq -r '.[1:-1][] | select(.is_original) | .path' | sort > original_files
     # Now we only need to combine both files:
     $ cat unique_files original_files
+    # Alternatively in one step:
+    $ rmlint t -o json -c json:unique |  jq -r '.[1:-1][] | select(.is_original) | .path' | sort > original_files
 
 Filter by regular expressions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

gui/shredder/application.py

@@ -79,13 +79,6 @@ def __init__(self, options):
         self.cmd_opts = options
         self.settings = self.win = None
 
-        # Check compile time features of rmlint that we need later.
-        if not have_feature('replay'):
-            LOGGER.error('No support for +replay in rmlint binary.')
-            LOGGER.error('Please recompile with --with-json-glib…')
-            LOGGER.error('…and `json-glib-1.0` installed on your system.')
-            sys.exit(-1)
-
     def do_activate(self, **kw):
         Gtk.Application.do_activate(self, **kw)
         self.win.present()

lib/SConscript

@@ -20,7 +20,6 @@ def build_config_template(target, source, env):
             INSTALL_PREFIX=GetOption('actual_prefix') or GetOption('prefix'),
             HAVE_LIBINTL=env['HAVE_GETTEXT'],
             HAVE_LIBELF=env['HAVE_LIBELF'],
-            HAVE_JSON_GLIB=env['HAVE_JSON_GLIB'],
             HAVE_GIO_UNIX=env['HAVE_GIO_UNIX'],
             HAVE_FIEMAP=env['HAVE_FIEMAP'],
             HAVE_XATTR=env['HAVE_XATTR'],

lib/cfg.c

@@ -27,6 +27,7 @@
 #include <unistd.h>
 
 #include "cfg.h"
+#include "logger.h"
 #include "utilities.h"
 
 static void rm_path_free(RmPath *rmpath) {

lib/cfg.h

@@ -83,7 +83,6 @@ typedef struct RmCfg {
     gboolean write_cksum_to_xattr;
     gboolean read_cksum_from_xattr;
     gboolean clear_xattr_fields;
-    gboolean write_unfinished;
     gboolean build_fiemap;
     gboolean use_buffered_read;
     gboolean fake_fiemap;
@@ -93,6 +92,8 @@ typedef struct RmCfg {
     gboolean read_stdin;
     gboolean read_stdin0;
     gboolean backup;
+    gboolean hash_uniques;
+    gboolean hash_unmatched;
 
     int permissions;
 
@@ -178,13 +179,6 @@ typedef struct RmCfg {
      * (or directories)
      */
     gboolean run_equal_mode;
-    /* --dedupe options */
-    bool dedupe;
-    bool dedupe_check_xattr;
-    bool dedupe_readonly;
-
-    /* for --is-reflink option */
-    bool is_reflink;
 
     /* don't use sse accelerations */
     bool no_sse;

lib/checksum.c

@@ -47,6 +47,7 @@
 #include "checksums/sha3/sha3.h"
 #include "checksums/xxhash/xxhash.h"
 
+#include "logger.h"
 #include "utilities.h"
 
 #define _RM_CHECKSUM_DEBUG 0