Merge pull request #1930 from billsacks/query_testtypes

Add option to define test types in query_testlists
Test suite: scripts_regression_tests on cheyenne
Test baseline: N/A
Test namelist changes: none
Test status: bit for bit

Fixes #1929

User interface changes?: adds --define-testtypes option to query_testlists

Update gh-pages html (Y/N)?: N

Code review: Jedwards jgfouca

config/config_tests.xml

@@ -16,6 +16,9 @@ The following are the test functionality categories:
   11) other component-specific tests
 
 NOTES:
+- specifying the attribute INFRASTRUCTURE_TEST="TRUE" declares the given
+  test type to be an infrastructure-only test that is only meant to be
+  run by scripts_regression_tests.
 - unless otherwise noted everything is run in one executable directory
 - suffix: denotes the component history file suffixes that are added as part of the test
 - IOP test is done along with regular tests - not as a separate test
@@ -336,7 +339,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
   </test>
 
   <test NAME="HOMME">
-    <DESC>Run homme tests.</DESC>
+    <DESC>Run homme tests. Only works with the ACME version of the atmosphere component.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
     <STOP_N>11</STOP_N>
@@ -354,7 +357,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTBUILDFAIL">
+  <test NAME="TESTBUILDFAIL" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Insta-fail build step.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -363,7 +366,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTBUILDFAILEXC">
+  <test NAME="TESTBUILDFAILEXC" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Insta-fail build step by failing to init.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -372,7 +375,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTRUNFAIL">
+  <test NAME="TESTRUNFAIL" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Insta-fail run step.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -381,7 +384,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTRUNFAILEXC">
+  <test NAME="TESTRUNFAILEXC" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Insta-fail run step via exception.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -390,7 +393,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTRUNPASS">
+  <test NAME="TESTRUNPASS" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Insta-pass run step.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -399,7 +402,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTMEMLEAKFAIL">
+  <test NAME="TESTMEMLEAKFAIL" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Insta-fail memleak step.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -408,7 +411,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTMEMLEAKPASS">
+  <test NAME="TESTMEMLEAKPASS" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Insta-pass memleak step.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -417,7 +420,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTRUNDIFF">
+  <test NAME="TESTRUNDIFF" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Produces a canned hist file.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -426,7 +429,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTTESTDIFF">
+  <test NAME="TESTTESTDIFF" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Simulates internal test diff (non baseline)</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -435,7 +438,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="TESTRUNSLOWPASS">
+  <test NAME="TESTRUNSLOWPASS" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. After 5 minutes of sleep, pass run step.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>ndays</STOP_OPTION>
@@ -444,7 +447,7 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
-  <test NAME="NODEFAIL">
+  <test NAME="NODEFAIL" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Tests restart upon detected node failure</DESC>
     <INFO_DBUG>1</INFO_DBUG>
     <STOP_OPTION>nsteps</STOP_OPTION>

scripts/lib/CIME/XML/tests.py

@@ -32,3 +32,22 @@ def get_test_node(self, testname):
         node = self.get_node("test",{"NAME":testname})
         logger.debug("Found {}".format(node.text))
         return node
+
+    def print_values(self, skip_infrastructure_tests=True):
+        """
+        Print each test type and its description.
+
+        If skip_infrastructure_tests is True, then this does not write
+        information for tests with the attribute
+        INFRASTRUCTURE_TEST="TRUE".
+        """
+        all_tests = self.get_nodes(nodename="test")
+        for one_test in all_tests:
+            if skip_infrastructure_tests:
+                infrastructure_test = one_test.get("INFRASTRUCTURE_TEST")
+                if (infrastructure_test is not None and
+                    infrastructure_test.upper() == "TRUE"):
+                    continue
+            name = one_test.get("NAME")
+            desc = one_test.find("DESC").text
+            print("{}: {}".format(name, desc))

scripts/query_testlists

@@ -4,7 +4,7 @@
 Script to query xml test lists, displaying all tests in human-readable form.
 
 Usage:
-   ./query_testlists [--show-options]
+   ./query_testlists [--show-options] [--define-testtypes]
       Display a list of tests
    ./query_testlists --count
       Count tests by category/machine/compiler
@@ -16,6 +16,7 @@ Usage:
 from __future__ import print_function
 from Tools.standard_script_setup import *
 from CIME.test_utils import get_tests_from_xml, test_to_string
+from CIME.XML.tests import Tests
 from CIME.utils import expect
 
 logger = logging.getLogger(__name__)
@@ -46,6 +47,11 @@ def parse_command_line(args, description):
                         "(wallclock time, memory leak tolerance, etc.). "
                         "(Has no effect with --list or --count options.)")
 
+    parser.add_argument("--define-testtypes", action="store_true",
+                        help="At the top of the list of tests, define "
+                        "all of the possible test types. "
+                        "(Has no effect with --list or --count options.)")
+
     parser.add_argument("--xml-category",
                         help="Only include tests in this category; default is all categories")
 
@@ -61,14 +67,37 @@ def parse_command_line(args, description):
 
     args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
 
-    expect(not(args.count and args.list_type),
-           "Cannot specify both --count and --list arguments.")
+    _check_argument_compatibility(args)
 
     if args.list_type:
         _process_list_type(args)
 
     return args
 
+###############################################################################
+def _check_argument_compatibility(args):
+###############################################################################
+    """Ensures there are no incompatible arguments
+
+    If incompatible arguments are found, aborts with a helpful error
+    message.
+    """
+
+    expect(not(args.count and args.list_type),
+           "Cannot specify both --count and --list arguments.")
+
+    if args.count:
+        expect(not args.show_options,
+               "--show-options is incompatible with --count")
+        expect(not args.define_testtypes,
+               "--define-testtypes is incompatible with --count")
+
+    if args.list_type:
+        expect(not args.show_options,
+               "--show-options is incompatible with --list")
+        expect(not args.define_testtypes,
+               "--define-testtypes is incompatible with --list")
+
 ###############################################################################
 def _process_list_type(args):
 ###############################################################################
@@ -87,7 +116,7 @@ def _process_list_type(args):
         args.list_type = 'compiler'
 
 ###############################################################################
-def print_test_data(test_data, show_options):
+def print_test_data(test_data, show_options, define_testtypes):
 ###############################################################################
     """
     Args:
@@ -96,6 +125,14 @@ def print_test_data(test_data, show_options):
             - category: test category
     """
 
+    if define_testtypes:
+        print("#"*72)
+        print("Test types")
+        print("----------")
+        test_definitions = Tests()
+        test_definitions.print_values(skip_infrastructure_tests=True)
+        print("#"*72)
+
     categories = sorted(set([item['category'] for item in test_data]))
     max_category_len = max([len(category) for category in categories])
     max_test_len = max([len(item['name']) for item in test_data])
@@ -178,7 +215,7 @@ def _main_func(description):
     elif args.list_type:
         list_test_data(test_data, args.list_type)
     else:
-        print_test_data(test_data, args.show_options)
+        print_test_data(test_data, args.show_options, args.define_testtypes)
 
 if __name__ == "__main__":
     _main_func(__doc__)

scripts/tests/scripts_regression_tests.py

@@ -2287,8 +2287,8 @@ def _run_and_assert_query_testlist(self, extra_args=""):
         files = Files()
         testlist_drv = files.get_value("TESTS_SPEC_FILE", {"component":"drv"})
 
-        run_cmd_assert_result(self, "%s/query_testlists --xml-testlist %s %s"%
-                              (SCRIPT_DIR, testlist_drv, extra_args))
+        run_cmd_assert_result(self, "{}/query_testlists --xml-testlist {} {}".format(
+            SCRIPT_DIR, testlist_drv, extra_args))
 
     def test_query_testlists_runs(self):
         """Make sure that query_testlists runs successfully
@@ -2299,6 +2299,10 @@ def test_query_testlists_runs(self):
         """
         self._run_and_assert_query_testlist(extra_args="--show-options")
 
+    def test_query_testlists_define_testtypes_runs(self):
+        """Make sure that query_testlists runs successfully with the --define-testtypes argument"""
+        self._run_and_assert_query_testlist(extra_args="--define-testtypes")
+
     def test_query_testlists_count_runs(self):
         """Make sure that query_testlists runs successfully with the --count argument"""
         self._run_and_assert_query_testlist(extra_args="--count")