Add functionality to validate project groups in config

benchcab/config.py

@@ -91,6 +91,16 @@ def read_optional_key(config: dict):
             raise ValueError(msg)
         config["project"] = os.environ["PROJECT"]
 
+    # Directory List is obtained from Gadi Resources - https://opus.nci.org.au/display/Help/0.+Welcome+to+Gadi
+    data_dirs = ["/g/data", "/scratch"]
+    groups = list(
+        set([group for data_dir in data_dirs for group in os.listdir(data_dir)])
+    )
+
+    if config["project"] not in groups:
+        msg = f"Could not validate project name ({config['project']}): Check if project name is correct"
+        raise ConfigValidationError(msg)
+
     if "realisations" in config:
         for r in config["realisations"]:
             r["name"] = r.get("name")
@@ -152,8 +162,8 @@ def read_config(config_path: str) -> dict:
     """
     # Read configuration file
     config = read_config_file(config_path)
-    # Populate configuration dict with optional keys
-    read_optional_key(config)
     # Validate and return.
     validate_config(config)
+    # Populate configuration dict with optional keys
+    read_optional_key(config)
     return config

benchcab/data/test/config-optional.yml

@@ -1,5 +1,5 @@
 # Config with optional data
-project: optional
+project: hh5 
 
 fluxsite:
   experiment: AU-Tum

tests/test_config.py

@@ -13,7 +13,7 @@
 import benchcab.utils as bu
 
 # Temporarily set $PROJECT for testing module
-OPTIONAL_CONFIG_PROJECT = "tt1"
+OPTIONAL_CONFIG_PROJECT = "ks32"
 
 
 @pytest.fixture(autouse=True)
@@ -24,6 +24,13 @@ def _set_project_env_variable(monkeypatch):
         yield
 
 
+@pytest.fixture(autouse=True)
+def _set_project_validation_dirs():
+    with mock.patch("os.listdir") as mocked_listdir:
+        mocked_listdir.return_value = ["hh5", OPTIONAL_CONFIG_PROJECT]
+        yield
+
+
 @pytest.fixture()
 def config_str(request) -> str:
     """Provide relative YAML path string of data files."""
@@ -89,7 +96,7 @@ def all_optional_custom_config(default_only_config) -> dict:
     Reads from config-optional.yml
     """
     config = default_only_config | {
-        "project": "optional",
+        "project": "hh5",
         "fluxsite": {
             "experiment": "AU-Tum",
             "multiprocess": False,