Support list for TAAT() PASS_REGULAR_EXPRESSION and FAIL_REGULAR_EXPR…

…ESSION (TriBITSPub#464)

Turns out that the tribits_add_advanced_test() test-block arguments
PASS_REGULAR_EXPRESSION and FAIL_REGULAR_EXPRESSION did not support taking in
a list of regexes like the built-in CMake CTest properties of the same names.
This commit updates TAAT() to allow a list of arguments and they behave the
exact same way as the built-in properties (i.e. matching any).

I also updated the documentation to show how to pass in a list of regexes
correctly for PASS_REGULAR_EXPRESSION, FAIL_REGULAR_EXPRESSION,
FINAL_PASS_REGULAR_EXPRESSION and FINAL_FAIL_REGULAR_EXPRESSION.

I also fixed the documentation for tribits_add_test() as well for
PASS_REGULAR_EXPRESSION and FAIL_REGULAR_EXPRESSION.

Basically, a well constructed CMake function should almost never take in a
list of arguments expecting an explicit semi-colon.  This was part of the
learning in PR TriBITSPub#464 with how to treat semi-colons in CMake.

This commit was driven by a usage of FAIL_REGULAR_EXPRESSION in Trilinos while
testing updated TriBITS as part of TriBITSPub#299 when I realized the documentation was
wrong for how to deal with these lists of regexes.

test/core/CMakeLists.txt

@@ -49,7 +49,7 @@ tribits_add_advanced_test( TestingFunctionMacro_UnitTests
       -D${PROJECT_NAME}_TRIBITS_DIR=${${PROJECT_NAME}_TRIBITS_DIR}
       -P "${CMAKE_CURRENT_SOURCE_DIR}/TestingFunctionMacro_UnitTests.cmake"
     PASS_REGULAR_EXPRESSION_ALL
-      "Final UnitTests Result: num_run = 686"
+      "Final UnitTests Result: num_run = 695"
       "Final UnitTests Result: PASSED"
   )
 

test/core/CTestScriptsUnitTests/CMakeLists.txt

@@ -76,12 +76,19 @@ tribits_add_test( GenericDriver
   )
 
 tribits_add_test( GenericDriver
-  NAME CTestScripts_simple_pass_regex
+  NAME CTestScripts_simple_pass_regex_1
   ARGS "\"This is a crazy test!\" 1"
   PASS_REGULAR_EXPRESSION "[^=]This is a crazy test"
   NUM_MPI_PROCS 1
   )
 
+tribits_add_test( GenericDriver
+  NAME CTestScripts_simple_pass_regex_2
+  POSTFIX_AND_ARGS_0 test1 "This is a crazy test!" 1
+  PASS_REGULAR_EXPRESSION "does not match" "This is a crazy test"
+  NUM_MPI_PROCS 1
+  )
+
 tribits_add_test( GenericDriver
   NAME CTestScripts_simple_posfix_and_args_1
   POSTFIX_AND_ARGS_0 test1 "This is a crazy test!" 1
@@ -143,6 +150,14 @@ tribits_add_test( GenericDriver
   # "failed".  If you comment out FAIL_REGULAR_EXPRESSION, the final result
   # will be "failed".
 
+tribits_add_test( GenericDriver
+  NAME CTestScripts_pass_regular_expression_fail_regular_exprssion_2_will_fail
+  POSTFIX_AND_ARGS_0 test1 "This says PASS!  This says FAIL!" 1
+  PASS_REGULAR_EXPRESSION "This says PASS"
+  FAIL_REGULAR_EXPRESSION "Does not match" "This says FAIL"
+  WILL_FAIL
+  NUM_MPI_PROCS 1
+  )
 
 function(ctestscripts_fail_regular_exprssion_w_circular_ref_detection_tests)
 
@@ -206,13 +221,31 @@ tribits_add_advanced_test(
   )
 
 tribits_add_advanced_test(
-  CTestScripts_cmnd_1_args_1_print_msg_out_file
+  CTestScripts_cmnd_1_args_1_print_msg_out_file_pass_regex_1
   TEST_0 EXEC GenericDriver ARGS "This is a crazy test!" 5
     OUTPUT_FILE cmnd_1_args_1_print_msg_out_file_outputFile.out
     PASS_REGULAR_EXPRESSION "This is a crazy test"
   OVERALL_NUM_MPI_PROCS 1
   )
 
+tribits_add_advanced_test(
+  CTestScripts_cmnd_1_args_1_print_msg_out_file_pass_regex_2
+  TEST_0 EXEC GenericDriver ARGS "This is a crazy test!" 5
+    OUTPUT_FILE cmnd_1_args_1_print_msg_out_file_outputFile.out
+    PASS_REGULAR_EXPRESSION "does not match" "This is a crazy test"
+  OVERALL_NUM_MPI_PROCS 1
+  )
+
+tribits_add_advanced_test(
+  CTestScripts_cmnd_1_args_1_print_msg_out_file_fail_regex_2_will_fail
+  TEST_0 EXEC GenericDriver ARGS "This is a crazy test!" 0
+    OUTPUT_FILE cmnd_1_args_1_print_msg_out_file_outputFile.out
+    PASS_REGULAR_EXPRESSION "This is a crazy test"
+    FAIL_REGULAR_EXPRESSION "Does not match" "This is a crazy test"
+    WILL_FAIL
+  OVERALL_NUM_MPI_PROCS 1
+  )
+
 tribits_add_test(GenericDriver
   NAME CTestScripts_tat_args_with_semicolon
   ARGS "arg<sep>with<sep>a<sep>few<sep>semicolons 0"
@@ -346,7 +379,7 @@ tribits_add_advanced_test(
     ARGS "This is a crazy test\n\nThis is not the best test" 5
     OUTPUT_FILE cmnd_1_args_1_pass_regular_expression_all_compare_1_outputFile.out
     PASS_REGULAR_EXPRESSION_ALL "This is a crazy test" "This is not the best test"
-  FINAL_PASS_REGULAR_EXPRESSION "TEST_0: Pass criteria = Match REGEX {This is a crazy test} [^a]PASSED[^a]"
+  FINAL_PASS_REGULAR_EXPRESSION "TEST_0: Pass criteria = Match all REGEX {This is a crazy test} [^a]PASSED[^a]"
   OVERALL_NUM_MPI_PROCS 1
   )
 
@@ -356,7 +389,7 @@ tribits_add_advanced_test(
     ARGS "This is a crazy test\n\nThis is not the best test" 5
     OUTPUT_FILE cmnd_1_args_1_pass_regular_expression_all_compare_2_outputFile.out
     PASS_REGULAR_EXPRESSION_ALL "This is a crazy test" "This is not the best test"
-  FINAL_PASS_REGULAR_EXPRESSION "TEST_0: Pass criteria = Match REGEX {This is not the best test} [^a]PASSED[^a]"
+  FINAL_PASS_REGULAR_EXPRESSION "TEST_0: Pass criteria = Match all REGEX {This is not the best test} [^a]PASSED[^a]"
   OVERALL_NUM_MPI_PROCS 1
   )
 
@@ -805,7 +838,7 @@ tribits_add_advanced_test(
       "TEST_1: Result = PASSED"
       "This is Test File A"
       "TEST_2: Return code = 0"
-      "TEST_2: Pass criteria = Match REGEX .This is Test File A. .PASSED."
+      "TEST_2: Pass criteria = Match any REGEX .This is Test File A. .PASSED."
       "TEST_2: Result = PASSED"                                
       "OVERALL FINAL RESULT: TEST FAILED [(]TAATDriver_TAAT_COPY_FILES_TO_TEST_DIR_bad_file_name[)]"
   )
@@ -841,7 +874,7 @@ tribits_add_advanced_test(
       "TEST_3: Result = PASSED"
       "This is Test File A"
       "TEST_4: Return code = 0"
-      "TEST_4: Pass criteria = Match REGEX .This is Test File A. .PASSED."
+      "TEST_4: Pass criteria = Match any REGEX .This is Test File A. .PASSED."
       "TEST_4: Result = PASSED"                                
       "OVERALL FINAL RESULT: TEST FAILED [(]TAATDriver_TAAT_COPY_FILES_TO_TEST_DIR_bad_dest_dir[)]"
   )

test/core/TestingFunctionMacro_UnitTests.cmake

@@ -2214,6 +2214,32 @@ function(unittest_tribits_add_test_properties)
   unittest_has_substr_const(TRIBITS_SET_TEST_PROPERTIES_INPUT
     "PackageA_SomeExec;APPEND;PROPERTY;LABELS")
 
+  message("Test setting PASS_REGULAR_EXPRESSION")
+  tribits_add_test(${EXEN} PASS_REGULAR_EXPRESSION
+     "[^a-zA-Z0-9_]My first pass Regex" "*/second pass regex")
+  unittest_compare_const(MESSAGE_WRAPPER_INPUT
+    "-- ${PACKEXEN}: Added test (BASIC, PROCESSORS=1)!" )
+  unittest_has_substr_const(TRIBITS_SET_TEST_PROPERTIES_INPUT
+    "PackageA_SomeExec;PROPERTIES;PASS_REGULAR_EXPRESSION;[^a-zA-Z0-9_]My first pass Regex;*/second pass regex;")
+
+  message("Test setting FAIL_REGULAR_EXPRESSION")
+  tribits_add_test(${EXEN} FAIL_REGULAR_EXPRESSION
+     "[^a-zA-Z0-9_]My first fail Regex" "*/second fail regex")
+  unittest_compare_const(MESSAGE_WRAPPER_INPUT
+    "-- ${PACKEXEN}: Added test (BASIC, PROCESSORS=1)!" )
+  unittest_has_substr_const(TRIBITS_SET_TEST_PROPERTIES_INPUT
+    "PackageA_SomeExec;APPEND;PROPERTY;FAIL_REGULAR_EXPRESSION;[^a-zA-Z0-9_]My first fail Regex;*/second fail regex;")
+
+  message("Test setting WILL_FAIL")
+  tribits_add_test(${EXEN} WILL_FAIL FAIL_REGULAR_EXPRESSION
+     "[^a-zA-Z0-9_]1ST fail Regex" "*/2nd fail regex")
+  unittest_compare_const(MESSAGE_WRAPPER_INPUT
+    "-- ${PACKEXEN}: Added test (BASIC, PROCESSORS=1)!" )
+  unittest_has_substr_const(TRIBITS_SET_TEST_PROPERTIES_INPUT
+    "PackageA_SomeExec;PROPERTIES;WILL_FAIL;ON;")
+  unittest_has_substr_const(TRIBITS_SET_TEST_PROPERTIES_INPUT
+    "PackageA_SomeExec;APPEND;PROPERTY;FAIL_REGULAR_EXPRESSION;[^a-zA-Z0-9_]1ST fail Regex;*/2nd fail regex;")
+
   message("Test setting integer TIMEOUT with no scaling (not even defined)")
   tribits_add_test(${EXEN} TIMEOUT 200)
   unittest_compare_const(MESSAGE_WRAPPER_INPUT
@@ -3539,6 +3565,21 @@ function(unittest_tribits_add_advanced_test_properties)
   # tribits_add_advanced_test() that looks for it.  Otherwise, the outer
   # tribits_add_advanced_test() always thinks these unit tests pass!
 
+  message("Test setting FINAL_PASS_REGULAR_EXPRESSION")
+  tribits_add_advanced_test_unittest_reset()
+  tribits_add_advanced_test( TAAT_final_pass_regular_expression
+    TEST_0 CMND ${CMNDN} FINAL_PASS_REGULAR_EXPRESSION
+     "[^a-zA-Z0-9_]My first pass Regex" "*/second pass regex")
+  unittest_has_substr_const(TRIBITS_SET_TEST_PROPERTIES_INPUT
+    "PackageA_TAAT_final_pass_regular_expression;PROPERTIES;PASS_REGULAR_EXPRESSION;[^a-zA-Z0-9_]My first pass Regex;*/second pass regex")
+
+  message("Test setting FINAL_FAIL_REGULAR_EXPRESSION")
+  tribits_add_advanced_test( TAAT_final_fail_regular_expression
+    TEST_0 CMND ${CMNDN} FINAL_FAIL_REGULAR_EXPRESSION
+     "[^a-zA-Z0-9_]My first fail Regex" "*/second fail regex")
+  unittest_has_substr_const(TRIBITS_SET_TEST_PROPERTIES_INPUT
+    "PackageA_TAAT_final_fail_regular_expression;PROPERTIES;FAIL_REGULAR_EXPRESSION;[^a-zA-Z0-9_]My first fail Regex;*/second fail regex")
+
   message("Test setting non-integer TIMEOUT with no scaling (not even defined)")
   tribits_add_advanced_test_unittest_reset()
   tribits_add_advanced_test( TAAT_basic_cmnd_1_args_0
@@ -4514,4 +4555,4 @@ message("*** Determine final result of all unit tests")
 message("***\n")
 
 # Pass in the number of expected tests that must pass!
-unittest_final_result(686)
+unittest_final_result(695)

tribits/CHANGELOG.md

@@ -2,12 +2,28 @@
 ChangeLog for TriBITS
 ----------------------------------------
 
+## 2022-05-16:
+
+* **Added:** The function `tribits_add_advanced_test(`) now correctly accepts
+  a list of regexes for `PASS_REGULAR_EXPRESSION` and
+  `FAIL_REGULAR_EXPRESSION` and behave the same as the raw CTest properties of
+  the same names.
+
 ## 2022-03-10:
 
 * **Changed:** The `tribits_add_advanced_test()` command now correctly reports
   unparsed/unrecognized arguments.  This will break some sloppy usages. (One
   TriBITS test had to be fixed.)
 
+* **Changed:** The `tribits_add_test()` and `tribits_add_advanced_test()`
+  behave differently with data passed in with explicit colon arguments.  For
+  example, before `PASS_REGULAR_EXPRESSION "<regex0>;<regex1>"` could be used
+  to pass in a list of regular expressions but with the new handling of
+  function arguments, this now gets set as a single regex
+  `"<regex0>\\;<regex1>"` which is not the same.  The fix (that also works
+  with older versions of TriBITS) is to pass in multiple regexes as separate
+  arguments as `PASS_REGULAR_EXPRESSION "<regex0>" "<regex1>"`.
+
 * **Added:** The `tribits_add_test()`, `tribits_add_advanced_test()`, and
   `tribits_add_executable_and_test()` functions now allow handling semi-colons
   ';' in the quoted arguments to CMND/EXEC `ARGS` and `ENVIRONMENT` variable

tribits/core/package_arch/TribitsAddAdvancedTest.cmake

@@ -113,10 +113,10 @@ include(PrintVar)
 #      [OUTPUT_FILE <outputFile>]
 #      [NO_ECHO_OUTPUT]]
 #      [PASS_ANY
-#        | PASS_REGULAR_EXPRESSION "<regex>"
-#        | PASS_REGULAR_EXPRESSION_ALL "<regex1>" "<regex2>" ... "<regexn>"
+#        | PASS_REGULAR_EXPRESSION "<regex0>" "<regex1>" ...
+#        | PASS_REGULAR_EXPRESSION_ALL "<regex0>" "<regex1>" ...
 #        | STANDARD_PASS_OUTPUT ]
-#      [FAIL_REGULAR_EXPRESSION "<regex>"]
+#      [FAIL_REGULAR_EXPRESSION "<regex0>" "<regex1>" ...]
 #      [ALWAYS_FAIL_ON_NONZERO_RETURN | ALWAYS_FAIL_ON_ZERO_RETURN]
 #      [WILL_FAIL]
 #
@@ -493,17 +493,17 @@ include(PrintVar)
 #     that is to follow will determine pass or fail based on output from this
 #     command in some way.
 #
-#   ``PASS_REGULAR_EXPRESSION "<regex>"``
+#   ``PASS_REGULAR_EXPRESSION "<regex0>"  "<regex1>" ...``
 #
-#     If specified, the test command will be assumed to pass if it matches the
-#     given regular expression.  Otherwise, it is assumed to fail.  TIPS:
-#     Replace ';' with '[;]' or CMake will interpret this as an array element
-#     boundary.  To match '.', use '[.]'.
+#     If specified, the test command will be assumed to pass if it matches
+#     **any** of the given regular expressions.  Otherwise, it is assumed to
+#     fail.  TIPS: Replace ';' with '[;]' or CMake will interpret this as an
+#     array element boundary.  To match '.', use '[.]'.
 #
-#   ``PASS_REGULAR_EXPRESSION_ALL "<regex1>" "<regex2>" ... "<regexn>"``
+#   ``PASS_REGULAR_EXPRESSION_ALL "<regex0>" "<regex1>" ...``
 #
 #     If specified, the test command will be assumed to pass if the output
-#     matches all of the provided regular expressions.  Note that this is not
+#     matches **all** of the provided regular expressions.  Note that this is not
 #     a capability of raw ctest and represents an extension provided by
 #     TriBITS.  NOTE: It is critical that you replace ';' with '[;]' or CMake
 #     will interpret this as an array element boundary.
@@ -515,14 +515,14 @@ include(PrintVar)
 #     This as the result of directly passing in ``PASS_REGULAR_EXPRESSION
 #     "End Result: TEST PASSED"``.
 #
-#   ``FAIL_REGULAR_EXPRESSION "<regex>"``
+#   ``FAIL_REGULAR_EXPRESSION "<regex0>" "<regex1>" ...``
 #
-#     If specified, the test command will be assumed to fail if it matches the
-#     given regular expression.  Otherwise, it is assumed to pass.  This will
-#     be applied and take precedence over other above pass criteria.  For
-#     example, if even if ``PASS_REGULAR_EXPRESSION`` or
-#     ``PASS_REGULAR_EXPRESSION_ALL`` match, then the test will be marked as
-#     failed if this fail regex matches the output.
+#     If specified, the test command will be assumed to fail if it matches
+#     **any** of the given regular expressions.  This will be applied and take
+#     precedence over other above pass criteria.  For example, if even if
+#     ``PASS_REGULAR_EXPRESSION`` or ``PASS_REGULAR_EXPRESSION_ALL`` match,
+#     then the test will be marked as failed if this fail regex matches the
+#     output.
 #
 #   ``ALWAYS_FAIL_ON_NONZERO_RETURN``
 #
@@ -602,35 +602,54 @@ include(PrintVar)
 #
 # **Test case Pass/Fail (tribits_add_advanced_test())**
 #
+# The logic given below can be used to determine pass/fail criteria for a test
+# case both based on what is printed in the test output **and** the return
+# code for the test block command.  Raw CTest, as of version 3.23, does not
+# allow that.  With raw CTest, one can only set determine pass/fail based the
+# test output **or** the return code, but not both.  This make
+# `tribits_add_advanced_test()`_ more attractive to use than
+# `tribits_add_test()`_ or raw ``add_test()`` in cases where it is important
+# to check both.
+#
 # The logic for how pass/fail for a ``TEST_<IDX>`` ``EXEC`` or ``CMND`` case
 # is applied is given by::
 #
 #   # A) Apply first set of pass/fail logic
 #   TEST_CASE_PASSED = FALSE
 #   If PASS_ANY specified:
 #     TEST_CASE_PASSED = TRUE
-#   Else If PASS_REGULAR_EXPRESSION specified and "<regex>" matches:
-#     TEST_CASE_PASSED = TRUE
+#   Else If PASS_REGULAR_EXPRESSION is specified:
+#     For each "<regexi>" in PASS_REGULAR_EXPRESSION:
+#       If "<regexi>" matches STDOUT:
+#         TEST_CASE_PASSED = TRUE
+#       Endif
+#     Endforeach
 #   Else if PASS_REGULAR_EXPRESSION_ALL specified:
 #     TEST_CASE_PASSED = TRUE
-#     For each "<regexi>":
-#       If "<regixi>" does not match:
+#     For each "<regexi>" in PASS_REGULAR_EXPRESSION_ALL:
+#       If "<regexi>" does not match STDOUT:
 #         TEST_CASE_PASSED = FALSE
+#       Endif
+#     Endforeach
 #   Else
 #     If command return code == 0:
 #       TEST_CASE_PASSED = TRUE
 #     Endif
 #   Endif
 #
 #   # B) Check for failing regex matching?
-#   If FAIL_REGULAR_EXPRESSION specified and "<regex>" matches:
-#     TEST_CASE_PASSED = FALSE
+#   If FAIL_REGULAR_EXPRESSION specified:
+#     For each "<regexi>" in FAIL_REGULAR_EXPRESSION:
+#       If "<regexi>" matches STDOUT:
+#         TEST_CASE_PASSED = FALSE
+#       Endif
+#     Endforeach
 #   Endif
 #
 #   # C) Check for return code always 0 or !=0?
 #   If ALWAYS_FAIL_ON_NONZERO_RETURN specified and return code != 0:
 #     TEST_CASE_PASSED = FALSE
-#   ElseIf ALWAYS_FAIL_ON_ZERO_RETURN specified and return code == 0:
+#   Else If ALWAYS_FAIL_ON_ZERO_RETURN specified and return code == 0:
 #     TEST_CASE_PASSED = FALSE
 #   Endif
 #
@@ -643,6 +662,13 @@ include(PrintVar)
 #     Endif
 #   Endif
 #
+# Note that the above is the exact same logic that CTest uses to determine
+# pass/fail w.r.t. to the CTest properties ``PASS_REGULAR_EXPRESSION``,
+# ``FAIL_REGULAR_EXPRESSION`` and ``WILL_FAIL``.  (It is just that raw
+# CMake/CTest, as of version 3.23, does not support any pass/fail criteria
+# like ``PASS_REGULAR_EXPRESSION_ALL`` or
+# ``ALWAYS_FAIL_ON_NONZERO_RETURN``/``ALWAYS_FAIL_ON_ZERO_RETURN``.)
+#
 # .. _Overall Pass/Fail (tribits_add_advanced_test()):
 #
 # **Overall Pass/Fail (tribits_add_advanced_test())**
@@ -653,15 +679,26 @@ include(PrintVar)
 #
 # However, this can be changed by setting one of the following optional arguments:
 #
-#   ``FINAL_PASS_REGULAR_EXPRESSION <regex>``
+#   ``FINAL_PASS_REGULAR_EXPRESSION "<regex0>" "<regex1>" ...``
 #
 #     If specified, the test will be assumed to pass if the output matches
-#     ``<regex>``.  Otherwise, it will be assumed to fail.
+#     **any** of the provided regular expressions ``<regexi>``.  Otherwise, it
+#     will be assumed to fail.  (Sets the CTest property
+#     ``PASS_REGULAR_EXPRESSION`` for the overall test.)
 #
-#   ``FINAL_FAIL_REGULAR_EXPRESSION <regex>``
+#   ``FINAL_FAIL_REGULAR_EXPRESSION "<regex0>" "<regex1>" ...``
 #
 #     If specified, the test will be assumed to fail if the output matches
-#     ``<regex>``.  Otherwise, it will be assumed to fail.
+#     **any** of the provided regular expressions ``<regexi>`` regardless if
+#     other criteria would have the test passing.  (Sets the CTest property
+#     ``FAIL_REGULAR_EXPRESSION`` for the overall test.)
+#
+# **NOTE:** It is **not** recommended to set ``FINAL_PASS_REGULAR_EXPRESSION``
+# or ``FINAL_FAIL_REGULAR_EXPRESSION`` directly, but instead to determine
+# pass/fail for each test case individually as described in `TEST_<idx>
+# EXEC/CMND Test Blocks and Arguments (tribits_add_advanced_test())`_ and
+# `Test case Pass/Fail (tribits_add_advanced_test())`_.  Otherwise, the test
+# will confuse most people and the output behavior will seem very strange.
 #
 # .. _Argument Parsing and Ordering (tribits_add_advanced_test()):
 #