add buffersize argument to rdfind (slightly modified version of trollkarlen's MR)

Makefile.am

@@ -18,7 +18,8 @@ TESTS=testcases/largefilesupport.sh \
       testcases/verify_deterministic_operation.sh \
       testcases/checksum_options.sh \
       testcases/md5collisions.sh \
-      testcases/sha1collisions.sh
+      testcases/sha1collisions.sh \
+      testcases/checksum_buffersize.sh
 
 AUXFILES=testcases/common_funcs.sh \
          testcases/md5collisions/letter_of_rec.ps \

testcases/checksum_buffersize.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+# Test that selection of buffersizes works as expected.
+
+set -e
+. "$(dirname "$0")/common_funcs.sh"
+
+reset_teststate
+
+TEST_DIR=buffersizes_test
+mkdir -p "$TEST_DIR"
+
+make_test_files() {
+    dbgecho "creating test files in $TEST_DIR"
+    head -c 1000000 /dev/zero >"$TEST_DIR/a"
+    cp "$TEST_DIR/a" "$TEST_DIR/b"
+    cp "$TEST_DIR/a" "$TEST_DIR/c"
+    cp "$TEST_DIR/a" "$TEST_DIR/d"
+    cp "$TEST_DIR/a" "$TEST_DIR/e"
+}
+
+dbgecho "check so all buffersizes behave the same"
+
+# disables only run once shellscheck
+# shellcheck disable=SC2043
+for checksumtype in sha256; do
+    i=1
+    while :; do
+        if [ $i -gt 128 ]; then
+            break
+        fi
+        i="$((i*2))"
+        make_test_files
+        dbgecho "testing buffersize $((i*1024))"
+        dbgecho "testing $checksumtype"
+        # Fix this properly by making rdfind to array and use "${rdfind[@]}"
+        # this requires bash not sh
+        # shellcheck disable=SC2086
+        $rdfind -buffersize $((i*1024)) -checksum "$checksumtype" -deleteduplicates true "$TEST_DIR" >/dev/null
+        [ -e "$TEST_DIR/a" ]
+        [ ! -e "$TEST_DIR/b" ]
+        [ ! -e "$TEST_DIR/c" ]
+        [ ! -e "$TEST_DIR/d" ]
+        [ ! -e "$TEST_DIR/e" ]
+    done
+done

testcases/checksum_buffersize_speedtest.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+# Performance test for checksumming with different buffersizes. Not meant
+# to be run for regular testing.
+
+set -e
+. "$(dirname "$0")/common_funcs.sh"
+
+reset_teststate
+
+TEST_DIR=buffersizes_speedtest
+mkdir -p "$TEST_DIR"
+
+make_test_files() {
+    dbgecho "creating test files in $TEST_DIR/bigfiles"
+    mkdir -p "$TEST_DIR/bigfiles"
+    head -c $((1024*1024*500)) /dev/zero >"$TEST_DIR/bigfiles/a"
+    for f in b c d e; do
+      cp "$TEST_DIR/bigfiles/a" "$TEST_DIR/bigfiles/$f"
+    done
+    dbgecho "creating test files in $TEST_DIR/smallfiles"
+    mkdir -p "$TEST_DIR/smallfiles"
+    (cd "$TEST_DIR/smallfiles"; head -c100000000 /dev/zero |split --bytes 1000)
+}
+
+dbgecho "run speed test for all shecksums and buffersizes"
+
+make_test_files
+
+cat /dev/null >"$TEST_DIR/results.tsv"
+for filesize in big small; do
+for checksumtype in md5 sha1; do
+    i=1
+    while :; do
+        if [ $i -gt 4096 ]; then
+            break
+        fi
+        # Fix this properly by making rdfind to array and use "${rdfind[@]}"
+        # this requires bash not sh
+        # shellcheck disable=SC2086
+        dbgecho "testing $checksumtype $i kB buffersize"
+        # shellcheck disable=SC2086
+        /usr/bin/time --append --output=$TEST_DIR/results.tsv -f "$filesize\t$i\t$checksumtype\t%e\t%M\t%C" $rdfind -buffersize $((i*1024)) -checksum "$checksumtype" -dryrun true -deleteduplicates true "$TEST_DIR/${filesize}files" >/dev/null 2>&1
+        i="$((i*2))"
+done
+done
+done
+cat "$TEST_DIR/results.tsv"
+