diff --git a/config/cesm/config_inputdata.xml b/config/cesm/config_inputdata.xml index b61dcb5778a..2cb2d6781ba 100644 --- a/config/cesm/config_inputdata.xml +++ b/config/cesm/config_inputdata.xml @@ -40,6 +40,10 @@ NEON Tower data for datm wget
https://s3.data.neonscience.org/neon-ncar/NEON/
+ + + + ../listing.csv diff --git a/config/xml_schemas/config_inputdata.xsd b/config/xml_schemas/config_inputdata.xsd index 88bab745216..fee8e973c10 100644 --- a/config/xml_schemas/config_inputdata.xsd +++ b/config/xml_schemas/config_inputdata.xsd @@ -8,12 +8,16 @@ - + + + + + diff --git a/scripts/lib/CIME/XML/inputdata.py b/scripts/lib/CIME/XML/inputdata.py index 626b0089302..f6c033bb169 100644 --- a/scripts/lib/CIME/XML/inputdata.py +++ b/scripts/lib/CIME/XML/inputdata.py @@ -32,6 +32,11 @@ def get_next_server(self, attributes=None): chksum_file = None ic_filepath = None servernodes = self.get_children("server", attributes=attributes) + + # inventory is a CSV list of available data files and the valid date for each + # expected format is pathtofile,YYYY-MM-DD HH:MM:SS + # currently only used for NEON tower data + inventory = None if not attributes: servernodes = [x for x in servernodes if not self.attrib(x)] @@ -52,6 +57,10 @@ def get_next_server(self, attributes=None): unode = self.get_optional_child("user", root = self._servernode) if unode: user = self.text(unode) + invnode = self.get_optional_child("inventory", root = self._servernode) + if invnode: + inventory = self.text(invnode) + pnode = self.get_optional_child("password", root = self._servernode) if pnode: passwd = self.text(pnode) @@ -62,4 +71,4 @@ def get_next_server(self, attributes=None): if icnode: ic_filepath = self.text(icnode) - return protocol, address, user, passwd, chksum_file, ic_filepath + return protocol, address, user, passwd, chksum_file, ic_filepath, inventory diff --git a/scripts/lib/CIME/case/check_input_data.py b/scripts/lib/CIME/case/check_input_data.py index 54a9276ed2c..32bfffbb91e 100644 --- a/scripts/lib/CIME/case/check_input_data.py +++ b/scripts/lib/CIME/case/check_input_data.py @@ -22,7 +22,7 @@ def _download_checksum_file(rundir): chksum_found = False # download and merge all available chksum files. while protocol is not None: - protocol, address, user, passwd, chksum_file,_ = inputdata.get_next_server() + protocol, address, user, passwd, chksum_file,_,_ = inputdata.get_next_server() if protocol not in vars(CIME.Servers): logger.info("Client protocol {} not enabled".format(protocol)) continue @@ -190,7 +190,7 @@ def _downloadfromserver(case, input_data_root, data_list_dir, attributes=None): input_data_root = case.get_value('DIN_LOC_ROOT') while not success and protocol is not None: - protocol, address, user, passwd, _, ic_filepath = inputdata.get_next_server(attributes=attributes) + protocol, address, user, passwd, _, ic_filepath, _ = inputdata.get_next_server(attributes=attributes) logger.info("Checking server {} with protocol {}".format(address, protocol)) success = case.check_input_data(protocol=protocol, address=address, download=True, input_data_root=input_data_root, @@ -352,9 +352,24 @@ def check_input_data(case, protocol="svn", address=None, input_data_root=None, d # proceed if not os.path.exists(full_path): print("Model {} missing file {} = '{}'".format(model, description, full_path)) + # Data download path must be DIN_LOC_ROOT, DIN_LOC_IC or RUNDIR + + rundir = case.get_value("RUNDIR") if download: - logger.warning(" Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root)) - no_files_missing = False + if full_path.startswith(rundir): + filepath = os.path.dirname(full_path) + if not os.path.exists(filepath): + logger.info("Creating directory {}".format(filepath)) + os.makedirs(filepath) + tmppath = full_path[len(rundir)+1:] + success = _download_if_in_repo(server, os.path.join(rundir,"inputdata"), + tmppath[10:], + isdirectory=isdirectory, ic_filepath='/') + no_files_missing = success + else: + logger.warning(" Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root)) + else: + no_files_missing = False else: logger.debug(" Found input file: '{}'".format(full_path)) else: diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py index 90d8e450c2e..25980853906 100644 --- a/scripts/lib/CIME/test_scheduler.py +++ b/scripts/lib/CIME/test_scheduler.py @@ -484,10 +484,15 @@ def _create_newcase_phase(self, test): testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component}) test_mod_file = os.path.join(testmods_dir, component, modspath) + # if no testmod is found check if a usermod of the same name exists and + # use it if it does. if not os.path.exists(test_mod_file): - error = "Missing testmod file '{}'".format(test_mod_file) - self._log_output(test, error) - return False, error + usermods_dir = files.get_value("USER_MODS_DIR", {"component": component}) + test_mod_file = os.path.join(usermods_dir, modspath) + if not os.path.exists(test_mod_file): + error = "Missing testmod file '{}', checked {} and {}".format(modspath, testmods_dir, usermods_dir) + self._log_output(test, error) + return False, error create_newcase_cmd += " --user-mods-dir {}".format(test_mod_file)