ci: Also run the dejagnu tests from GNU findutils

uutils · Feb 8, 2022 · 43027d0 · 43027d0
1 parent 5c45b30
commit 43027d0
Show file tree

Hide file tree

Showing 3 changed files with 81 additions and 44 deletions.
diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml
@@ -38,38 +38,13 @@ jobs:
     - name: Extract testing info
       shell: bash
       run: |
-        LOG_FILE=findutils.gnu/tests/test-suite.log
-        if test -f "$LOG_FILE"; then
-          TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p"  "$LOG_FILE"|tr -d '\r'|head -n1)
-          PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
-          SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
-          FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
-          XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
-          ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
-          if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then
-              echo "Error in the execution, failing early"
-              exit 1
-          fi
-          output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR"
-          echo "${output}"
-          if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi
-          jq -n \
-                --arg date "$(date --rfc-email)" \
-                --arg sha "$GITHUB_SHA" \
-                --arg total "$TOTAL" \
-                --arg pass "$PASS" \
-                --arg skip "$SKIP" \
-                --arg fail "$FAIL" \
-                --arg xpass "$XPASS" \
-                --arg error "$ERROR" \
-                '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > gnu-result.json
-        else
-          echo "::error ::Failed to get summary of test results"
-        fi
     - uses: actions/upload-artifact@v2
       with:
         name: gnu-test-report
-        path: findutils.gnu/tests/**/*.log
+        path: |
+          findutils.gnu/find/testsuite/*.log
+          findutils.gnu/xargs/testsuite/*.log
+          findutils.gnu/tests/**/*.log
     - uses: actions/upload-artifact@v2
       with:
         name: gnu-result
@@ -95,18 +70,9 @@ jobs:
     - name: Compare failing tests against master
       shell: bash
       run: |
-            OLD_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" dl/test-suite.log | sort)
-            NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" findutils.gnu/tests/test-suite.log | sort)
-            for LINE in $OLD_FAILING; do
-              if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then
-                echo "::warning ::Congrats! The gnu test $LINE is now passing!"
-              fi
-            done
-            for LINE in $NEW_FAILING; do
-              if ! grep -Fxq $LINE<<<"$OLD_FAILING"; then
-                echo "::error ::gnu test failed: $LINE. $LINE is passing on 'main'. Maybe you have to rebase?"
-              fi
-            done
+        mkdir -p ./logs
+        cp findutils.gnu/{tests,{find,xargs}/testsuite}/*.log ./logs
+        ./findutils/util/diff-gnu.sh dl logs
     - name: Compare against main results
       shell: bash
       run: |

diff --git a/util/build-gnu.sh b/util/build-gnu.sh
@@ -1,9 +1,10 @@
 #!/bin/bash
 
 set -e
+
 if test ! -d ../findutils.gnu; then
     echo "Could not find ../findutils.gnu"
-    echo "git clone  https://git.savannah.gnu.org/git/findutils.git findutils.gnu"
+    echo "git clone https://git.savannah.gnu.org/git/findutils.git findutils.gnu"
     exit 1
 fi
 
@@ -16,7 +17,7 @@ cp target/release/xargs ../findutils.gnu/xargs.rust
 cd ../findutils.gnu
 if test ! -f configure; then
     ./bootstrap
-    ./configure --quiet --disable-gcc-warnings
+    ./configure --quiet
     make -j "$(nproc)"
 fi
 
@@ -30,4 +31,55 @@ if test -n "$1"; then
 fi
 
 # Run the tests
-make check-TESTS $RUN_TEST
+make check-TESTS $RUN_TEST || :
+make -C find/testsuite check || :
+make -C xargs/testsuite check || :
+
+PASS=0
+SKIP=0
+FAIL=0
+XPASS=0
+ERROR=0
+
+LOG_FILE=./find/testsuite/find.log
+if test -f "$LOG_FILE"; then
+    ((PASS += $(sed -En 's/# of expected passes\s*//p' "$LOG_FILE"))) || :
+    ((FAIL += $(sed -En 's/# of unexpected failures\s*//p' "$LOG_FILE"))) || :
+fi
+
+LOG_FILE=./xargs/testsuite/xargs.log
+if test -f "$LOG_FILE"; then
+    ((PASS += $(sed -En 's/# of expected passes\s*//p' "$LOG_FILE"))) || :
+    ((FAIL += $(sed -En 's/# of unexpected failures\s*//p' "$LOG_FILE"))) || :
+fi
+
+((TOTAL = PASS + FAIL)) || :
+
+LOG_FILE=./tests/test-suite.log
+if test -f "$LOG_FILE"; then
+    ((TOTAL += $(sed -n "s/.*# TOTAL: \(.*\)/\1/p"  "$LOG_FILE" | tr -d '\r' | head -n1))) || :
+    ((PASS += $(sed -n "s/.*# PASS: \(.*\)/\1/p" "$LOG_FILE" | tr -d '\r' | head -n1))) || :
+    ((SKIP += $(sed -n "s/.*# SKIP: \(.*\)/\1/p" "$LOG_FILE" | tr -d '\r' | head -n1))) || :
+    ((FAIL += $(sed -n "s/.*# FAIL: \(.*\)/\1/p" "$LOG_FILE" | tr -d '\r' | head -n1))) || :
+    ((XPASS += $(sed -n "s/.*# XPASS: \(.*\)/\1/p" "$LOG_FILE" | tr -d '\r' | head -n1))) || :
+    ((ERROR += $(sed -n "s/.*# ERROR: \(.*\)/\1/p" "$LOG_FILE" | tr -d '\r' | head -n1))) || :
+fi
+
+if ((TOTAL <= 1)); then
+    echo "Error in the execution, failing early"
+    exit 1
+fi
+
+output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR"
+echo "${output}"
+if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi
+jq -n \
+   --arg date "$(date --rfc-email)" \
+   --arg sha "$GITHUB_SHA" \
+   --arg total "$TOTAL" \
+   --arg pass "$PASS" \
+   --arg skip "$SKIP" \
+   --arg fail "$FAIL" \
+   --arg xpass "$XPASS" \
+   --arg error "$ERROR" \
+   '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > ../gnu-result.json
diff --git a/util/diff-gnu.sh b/util/diff-gnu.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu
+
+export LC_COLLATE=C
+
+# Extract the failing test lines from log files
+failing_tests() {
+    sed -En 's/FAIL: ([^,:]*)[,:].*/\1/p' "$@" | sort
+}
+
+comm -3 <(failing_tests $1/*.log) <(failing_tests $2/*.log) | tr '\t' ',' | while IFS=, read old new foo; do
+    if [ -n "$old" ]; then
+        echo "::warning ::Congrats! The GNU test $old is now passing!"
+    fi
+    if [ -n "$new" ]; then
+        echo "::error ::GNU test failed: $new. $new is passing on 'main'. Maybe you have to rebase?"
+    fi
+done