diff --git a/config/cesm/config_inputdata.xml b/config/cesm/config_inputdata.xml
index b61dcb5778a..2cb2d6781ba 100644
--- a/config/cesm/config_inputdata.xml
+++ b/config/cesm/config_inputdata.xml
@@ -40,6 +40,10 @@
NEON Tower data for datm
wget
https://s3.data.neonscience.org/neon-ncar/NEON/
+
+
+
+ ../listing.csv
diff --git a/config/xml_schemas/config_inputdata.xsd b/config/xml_schemas/config_inputdata.xsd
index 88bab745216..fee8e973c10 100644
--- a/config/xml_schemas/config_inputdata.xsd
+++ b/config/xml_schemas/config_inputdata.xsd
@@ -8,12 +8,16 @@
-
+
+
+
+
+
diff --git a/scripts/lib/CIME/XML/inputdata.py b/scripts/lib/CIME/XML/inputdata.py
index 626b0089302..f6c033bb169 100644
--- a/scripts/lib/CIME/XML/inputdata.py
+++ b/scripts/lib/CIME/XML/inputdata.py
@@ -32,6 +32,11 @@ def get_next_server(self, attributes=None):
chksum_file = None
ic_filepath = None
servernodes = self.get_children("server", attributes=attributes)
+
+ # inventory is a CSV list of available data files and the valid date for each
+ # expected format is pathtofile,YYYY-MM-DD HH:MM:SS
+ # currently only used for NEON tower data
+ inventory = None
if not attributes:
servernodes = [x for x in servernodes if not self.attrib(x)]
@@ -52,6 +57,10 @@ def get_next_server(self, attributes=None):
unode = self.get_optional_child("user", root = self._servernode)
if unode:
user = self.text(unode)
+ invnode = self.get_optional_child("inventory", root = self._servernode)
+ if invnode:
+ inventory = self.text(invnode)
+
pnode = self.get_optional_child("password", root = self._servernode)
if pnode:
passwd = self.text(pnode)
@@ -62,4 +71,4 @@ def get_next_server(self, attributes=None):
if icnode:
ic_filepath = self.text(icnode)
- return protocol, address, user, passwd, chksum_file, ic_filepath
+ return protocol, address, user, passwd, chksum_file, ic_filepath, inventory
diff --git a/scripts/lib/CIME/case/check_input_data.py b/scripts/lib/CIME/case/check_input_data.py
index 54a9276ed2c..32bfffbb91e 100644
--- a/scripts/lib/CIME/case/check_input_data.py
+++ b/scripts/lib/CIME/case/check_input_data.py
@@ -22,7 +22,7 @@ def _download_checksum_file(rundir):
chksum_found = False
# download and merge all available chksum files.
while protocol is not None:
- protocol, address, user, passwd, chksum_file,_ = inputdata.get_next_server()
+ protocol, address, user, passwd, chksum_file,_,_ = inputdata.get_next_server()
if protocol not in vars(CIME.Servers):
logger.info("Client protocol {} not enabled".format(protocol))
continue
@@ -190,7 +190,7 @@ def _downloadfromserver(case, input_data_root, data_list_dir, attributes=None):
input_data_root = case.get_value('DIN_LOC_ROOT')
while not success and protocol is not None:
- protocol, address, user, passwd, _, ic_filepath = inputdata.get_next_server(attributes=attributes)
+ protocol, address, user, passwd, _, ic_filepath, _ = inputdata.get_next_server(attributes=attributes)
logger.info("Checking server {} with protocol {}".format(address, protocol))
success = case.check_input_data(protocol=protocol, address=address, download=True,
input_data_root=input_data_root,
@@ -352,9 +352,24 @@ def check_input_data(case, protocol="svn", address=None, input_data_root=None, d
# proceed
if not os.path.exists(full_path):
print("Model {} missing file {} = '{}'".format(model, description, full_path))
+ # Data download path must be DIN_LOC_ROOT, DIN_LOC_IC or RUNDIR
+
+ rundir = case.get_value("RUNDIR")
if download:
- logger.warning(" Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
- no_files_missing = False
+ if full_path.startswith(rundir):
+ filepath = os.path.dirname(full_path)
+ if not os.path.exists(filepath):
+ logger.info("Creating directory {}".format(filepath))
+ os.makedirs(filepath)
+ tmppath = full_path[len(rundir)+1:]
+ success = _download_if_in_repo(server, os.path.join(rundir,"inputdata"),
+ tmppath[10:],
+ isdirectory=isdirectory, ic_filepath='/')
+ no_files_missing = success
+ else:
+ logger.warning(" Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
+ else:
+ no_files_missing = False
else:
logger.debug(" Found input file: '{}'".format(full_path))
else:
diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py
index 90d8e450c2e..25980853906 100644
--- a/scripts/lib/CIME/test_scheduler.py
+++ b/scripts/lib/CIME/test_scheduler.py
@@ -484,10 +484,15 @@ def _create_newcase_phase(self, test):
testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
test_mod_file = os.path.join(testmods_dir, component, modspath)
+ # if no testmod is found check if a usermod of the same name exists and
+ # use it if it does.
if not os.path.exists(test_mod_file):
- error = "Missing testmod file '{}'".format(test_mod_file)
- self._log_output(test, error)
- return False, error
+ usermods_dir = files.get_value("USER_MODS_DIR", {"component": component})
+ test_mod_file = os.path.join(usermods_dir, modspath)
+ if not os.path.exists(test_mod_file):
+ error = "Missing testmod file '{}', checked {} and {}".format(modspath, testmods_dir, usermods_dir)
+ self._log_output(test, error)
+ return False, error
create_newcase_cmd += " --user-mods-dir {}".format(test_mod_file)