Merge pull request #4013 from jedwards4b/neon_data_chk

Changes the inputdata structure to add a field "inventory", currently this field is only used for NEON tower data and is a pointer to a CSV format file with filenames and modification dates for all available data on the server. This file was deemed necessary because query access to the data server is not allowed. This also extends the data download options so that if the data path begins with the case RUNDIR download is allowed. Previously only DIN_LOC_ROOT was allowed.

Test suite: Hand tested with SMS_D_Vnuopc_Mmpi-serial.CLM_USRDAT.I1PtClm51Bgc.cheyenne_intel.clm-NEON-NIWO , scripts_regression_tests.py

Test baseline:
Test namelist changes:
Test status: bit for bit,

Fixes

User interface changes?:

Update gh-pages html (Y/N)?:

config/cesm/config_inputdata.xml

@@ -40,6 +40,10 @@
     <comment> NEON Tower data for datm </comment>
     <protocol>wget</protocol>
     <address>https://s3.data.neonscience.org/neon-ncar/NEON/</address>
+    <!-- inventory is a CSV list of available data files and the valid date for each -->
+    <!-- expected format is pathtofile,YYYY-MM-DD HH:MM:SS -->
+    <!-- one per line where the date stamp indicates the last modification time of the file -->
+    <inventory>../listing.csv</inventory>
   </server>
 
 </inputdata>

config/xml_schemas/config_inputdata.xsd

@@ -8,12 +8,16 @@
           </xs:annotation>
           <xs:complexType>
             <xs:sequence>
-	      <xs:element type="xs:string" name="comment" minOccurs="0"/>
+              <xs:element type="xs:string" name="comment" minOccurs="0"/>
               <xs:element type="xs:string" name="protocol"/>
               <xs:element type="xs:string" name="address"/>
               <xs:element type="xs:string" name="user" minOccurs="0"/>
               <xs:element type="xs:string" name="password" minOccurs="0"/>
               <xs:element type="xs:string" name="checksum" minOccurs="0"/>
+                  <!-- inventory is a CSV list of available data files and the valid date for each -->
+                  <!-- expected format is pathtofile,YYYY-MM-DD HH:MM:SS -->
+                  <!-- it is currently only used for NEON tower data -->
+              <xs:element type="xs:string" name="inventory" minOccurs="0"/>
               <xs:element type="xs:string" name="ic_filepath" minOccurs="0"/>
             </xs:sequence>
           </xs:complexType>

scripts/lib/CIME/XML/inputdata.py

@@ -32,6 +32,11 @@ def get_next_server(self, attributes=None):
         chksum_file = None
         ic_filepath = None
         servernodes = self.get_children("server", attributes=attributes)
+
+        # inventory is a CSV list of available data files and the valid date for each
+        # expected format is pathtofile,YYYY-MM-DD HH:MM:SS
+        # currently only used for NEON tower data
+        inventory = None
         if not attributes:
             servernodes = [x for x in servernodes if not self.attrib(x)]
 
@@ -52,6 +57,10 @@ def get_next_server(self, attributes=None):
             unode = self.get_optional_child("user", root = self._servernode)
             if unode:
                 user =  self.text(unode)
+            invnode = self.get_optional_child("inventory", root = self._servernode)
+            if invnode:
+                inventory = self.text(invnode)
+
             pnode = self.get_optional_child("password", root = self._servernode)
             if pnode:
                 passwd =  self.text(pnode)
@@ -62,4 +71,4 @@ def get_next_server(self, attributes=None):
             if icnode:
                 ic_filepath =  self.text(icnode)
 
-        return protocol, address, user, passwd, chksum_file, ic_filepath
+        return protocol, address, user, passwd, chksum_file, ic_filepath, inventory

scripts/lib/CIME/case/check_input_data.py

@@ -22,7 +22,7 @@ def _download_checksum_file(rundir):
     chksum_found = False
     # download and merge all available chksum files.
     while protocol is not None:
-        protocol, address, user, passwd, chksum_file,_ = inputdata.get_next_server()
+        protocol, address, user, passwd, chksum_file,_,_ = inputdata.get_next_server()
         if protocol not in vars(CIME.Servers):
             logger.info("Client protocol {} not enabled".format(protocol))
             continue
@@ -190,7 +190,7 @@ def _downloadfromserver(case, input_data_root, data_list_dir, attributes=None):
         input_data_root = case.get_value('DIN_LOC_ROOT')
 
     while not success and protocol is not None:
-        protocol, address, user, passwd, _, ic_filepath = inputdata.get_next_server(attributes=attributes)
+        protocol, address, user, passwd, _, ic_filepath, _ = inputdata.get_next_server(attributes=attributes)
         logger.info("Checking server {} with protocol {}".format(address, protocol))
         success = case.check_input_data(protocol=protocol, address=address, download=True,
                                         input_data_root=input_data_root,
@@ -352,9 +352,24 @@ def check_input_data(case, protocol="svn", address=None, input_data_root=None, d
                         # proceed
                         if not os.path.exists(full_path):
                             print("Model {} missing file {} = '{}'".format(model, description, full_path))
+                            # Data download path must be DIN_LOC_ROOT, DIN_LOC_IC or RUNDIR
+
+                            rundir = case.get_value("RUNDIR")
                             if download:
-                                logger.warning("    Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
-                            no_files_missing = False
+                                if full_path.startswith(rundir):
+                                    filepath = os.path.dirname(full_path)
+                                    if not os.path.exists(filepath):
+                                        logger.info("Creating directory {}".format(filepath))
+                                        os.makedirs(filepath)
+                                    tmppath = full_path[len(rundir)+1:]
+                                    success = _download_if_in_repo(server, os.path.join(rundir,"inputdata"),
+                                                                   tmppath[10:],
+                                                                   isdirectory=isdirectory, ic_filepath='/')
+                                    no_files_missing = success
+                                else:
+                                    logger.warning("    Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
+                            else:
+                                no_files_missing = False
                         else:
                             logger.debug("  Found input file: '{}'".format(full_path))
                     else:

scripts/lib/CIME/test_scheduler.py

@@ -484,10 +484,15 @@ def _create_newcase_phase(self, test):
 
             testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
             test_mod_file = os.path.join(testmods_dir, component, modspath)
+            # if no testmod is found check if a usermod of the same name exists and
+            # use it if it does.
             if not os.path.exists(test_mod_file):
-                error = "Missing testmod file '{}'".format(test_mod_file)
-                self._log_output(test, error)
-                return False, error
+                usermods_dir = files.get_value("USER_MODS_DIR", {"component": component})
+                test_mod_file = os.path.join(usermods_dir, modspath)
+                if not os.path.exists(test_mod_file):
+                    error = "Missing testmod file '{}', checked {} and {}".format(modspath, testmods_dir, usermods_dir)
+                    self._log_output(test, error)
+                    return False, error
 
             create_newcase_cmd += " --user-mods-dir {}".format(test_mod_file)