Add cstest integration tests to CI

.github/workflows/CITest.yml

@@ -59,7 +59,7 @@ jobs:
       with:
         python-version: ${{ matrix.config.python-version }}
 
-    - name: prepare
+    - name: Prepare fuzzing
       run: |
         export LD_LIBRARY_PATH=`pwd`/tests/:$LD_LIBRARY_PATH
         wget https://github.com/groundx/capstonefuzz/raw/master/corpus/corpus-libFuzzer-capstone_fuzz_disasmnext-latest.zip
@@ -111,21 +111,16 @@ jobs:
       run: |
         sh suite/run_invalid_cstool.sh
 
-    - name: cstest (cmake)
+    - name: cstest integration tests (cmake)
       if: startsWith(matrix.config.build-system, 'cmake')
       run: |
-        python suite/cstest/cstest_report.py -D -d suite/MC
-        python suite/cstest/cstest_report.py -D -f suite/cstest/issues.cs
-        python suite/cstest/cstest_report.py -D -f tests/cs_details/issue.cs
+        cd suite/cstest/test
+        sh suite/run_invalid_cstool.sh
 
-    - name: cstest (make)
-      if: startsWith(matrix.config.build-system, 'make')
+    - name: cstest MC (cmake)
+      if: startsWith(matrix.config.build-system, 'cmake')
       run: |
-        cd suite/cstest && ./build_cstest.sh
-        python cstest_report.py -D -t build/cstest -d ../MC
-        python cstest_report.py -D -t build/cstest -f issues.cs
-        python cstest_report.py -D -t build/cstest -f ../../tests/cs_details/issue.cs
-        cd ../../
+        cstest tests/MC/
 
   Windows:
     runs-on: ${{ matrix.config.os }}

suite/cstest/test/README.md

@@ -0,0 +1 @@
+Integration tests of cstest

suite/cstest/test/integration_tests.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+# Copyright © 2024 Rot127 <[email protected]>
+# SPDX-License-Identifier: BSD-3
+
+import subprocess as sp
+
+
+def check(cmd: list[str], expected_stdout: str, expected_stderr: str, fail_msg: str):
+    result = sp.run(cmd, capture_output=True)
+    stderr = result.stderr.decode("utf8")
+    stdout = result.stdout.decode("utf8")
+    if expected_stderr and expected_stderr not in stderr:
+        print(f"STDERR mismatch: '{expected_stderr}' not in stderr")
+        print("\n###################### STDERR ######################\n")
+        print(stderr)
+        print("####################################################\n")
+        print(fail_msg)
+        exit(1)
+    if expected_stdout and expected_stdout not in stdout:
+        print(f"STDOUT mismatch: '{expected_stdout}' not in stdout")
+        print("\n###################### STDout ######################\n")
+        print(stdout)
+        print("####################################################\n")
+        print(fail_msg)
+        exit(1)
+
+
+def run_tests():
+    check(
+        ["cstest", "empty_test_file.yaml"],
+        expected_stderr="Failed to parse test file 'empty_test_file.yaml'",
+        expected_stdout="",
+        fail_msg="Failed the empty file test",
+    )
+
+    check(
+        ["cstest", "missing_madatory_field.yaml"],
+        expected_stderr="Error: 'Missing required mapping field'",
+        expected_stdout="",
+        fail_msg="Failed the mandatory field test",
+    )
+
+    check(
+        ["cstest", "invalid_test_file.yaml"],
+        expected_stderr="Error: 'libyaml parser error'",
+        expected_stdout="",
+        fail_msg="Failed the invalid test file test",
+    )
+
+    check(
+        ["cstest", "min_valid_test_file.yaml"],
+        expected_stdout="All tests succeeded.",
+        expected_stderr="",
+        fail_msg="Failed the minimal valid parsing test",
+    )
+
+    check(
+        ["cstest", "invalid_cs_input.yaml"],
+        expected_stderr="'ar' is not mapped to a capstone architecture.",
+        expected_stdout="",
+        fail_msg="Test: Invalid CS option failed",
+    )
+
+    check(
+        ["cstest", "invalid_cs_input.yaml"],
+        expected_stderr="[  ERROR   ] --- 0 != 0x1",
+        expected_stdout="",
+        fail_msg="Test: Wrong number of instruction disassembled failed",
+    )
+
+    check(
+        ["cstest", "invalid_cs_input.yaml"],
+        expected_stderr="Option: thum not used",
+        expected_stdout="",
+        fail_msg="Test: Invalid disassembly due to wrong option failed",
+    )
+
+    check(
+        ["cstest", "."],
+        expected_stdout="Test files found: 6",
+        expected_stderr="",
+        fail_msg="Test: Detecting file in directory failed.",
+    )
+
+
+if __name__ == "__main__":
+    run_tests()
+    print("All tests passed")
+    exit(0)

suite/cstest/test/invalid_cs_input.yaml

@@ -0,0 +1,29 @@
+test_cases:
+ -
+  input:
+    bytes: [ 0x05, 0xb0, 0xa0, 0xe1 ]
+    arch: "ar" # Wrong arch
+    options: ["arm"]
+  expected:
+    insns:
+      -
+        asm_text: "mov r11, r5"
+ -
+  input:
+    bytes: [ 0x06 ] # Wrong number of bytes.
+    arch: "aarch64"
+    options: []
+  expected:
+    insns:
+      -
+        asm_text: "mov r1, r6"
+ -
+  input:
+    bytes: [ 0xc2, 0xf3, 0x00, 0x8f ]
+    arch: "arm"
+    options: ["thum"] # Wrong mode
+  expected:
+    insns:
+      -
+          asm_text: "bxj r2"
+

suite/cstest/test/some_dir/some_other_dir/min_valid_test_file.yaml

@@ -0,0 +1,31 @@
+test_cases:
+ -
+  input:
+    bytes: [ 0x05, 0xb0, 0xa0, 0xe1 ]
+    arch: "arm"
+    options: ["arm"]
+  expected:
+    insns:
+      -
+        asm_text: "mov r11, r5"
+ -
+  input:
+    bytes: [ 0x06, 0x10, 0xa0, 0xe1 ]
+    arch: "arm"
+    options: ["arm"]
+  expected:
+    insns:
+      -
+        asm_text: "mov r1, r6"
+ -
+  input:
+    bytes: [ 0x06, 0x10, 0xa0, 0xe1, 0x05, 0xb0, 0xa0, 0xe1 ]
+    arch: "arm"
+    options: ["arm"]
+  expected:
+    insns:
+      -
+        asm_text: "mov r1, r6"
+      -
+        asm_text: "mov r11, r5"
+