Merge pull request #43 from CABLE-LSM/29-new-dev-feature

Add branch specific namelist settings

benchcab/bench_config.py

@@ -76,6 +76,12 @@ def check_config(config: dict):
                 f"The 'share_branch' field in realisation '{branch_id}' must be a "
                 "boolean."
             )
+        # the "patch" key is optional
+        if "patch" in branch_config and not isinstance(branch_config["patch"], dict):
+            raise TypeError(
+                f"The 'patch' field in realisation '{branch_id}' must be a "
+                "dictionary that is compatible with the f90nml python package."
+            )
 
 
 def get_science_config_id(key: str) -> str:
@@ -116,10 +122,11 @@ def read_config(config_path: str) -> dict:
 
     check_config(config)
 
-    # Add "revision" to each branch description if not provided with default value -1,
-    # i.e. HEAD of branch
     for branch in config['realisations'].values():
+        # Add "revision" key if not provided and set to default value -1, i.e. HEAD of branch
         branch.setdefault('revision', -1)
+        # Add "patch" key if not provided and set to default value {}
+        branch.setdefault('patch', {})
 
     return config
 

benchcab/task.py

@@ -38,12 +38,14 @@ def __init__(
         self,
         branch_id: int,
         branch_name: str,
+        branch_patch: dict,
         met_forcing_file: str,
         sci_conf_key: str,
         sci_config: dict
     ) -> None:
         self.branch_id = branch_id
         self.branch_name = branch_name
+        self.branch_patch = branch_patch
         self.met_forcing_file = met_forcing_file
         self.sci_conf_key = sci_conf_key
         self.sci_config = sci_config
@@ -108,9 +110,7 @@ def fetch_files(self, root_dir=internal.CWD):
         return self
 
     def adjust_namelist_file(self, root_dir=internal.CWD):
-        """Make necessary adjustments to the CABLE namelist file."""
-
-        task_dir = Path(root_dir, internal.SITE_TASKS_DIR, self.get_task_name())
+        """Sets the base settings in the CABLE namelist file for this task."""
 
         patch_nml = {
             "cable": {
@@ -135,15 +135,27 @@ def adjust_namelist_file(self, root_dir=internal.CWD):
 
         patch_nml["cable"].update(self.sci_config)
 
+        self.patch_namelist_file(patch_nml, root_dir=root_dir)
+
+        return self
+
+    def patch_namelist_file(self, patch: dict, root_dir=internal.CWD):
+        """Writes a patch to the CABLE namelist file for this task.
+
+        The `patch` dictionary must comply with the `f90nml` api.
+        """
+
+        task_dir = Path(root_dir, internal.SITE_TASKS_DIR, self.get_task_name())
+
         cable_nml = f90nml.read(str(task_dir / internal.CABLE_NML))
         # remove namelist file as f90nml cannot write to an existing file
         os.remove(str(task_dir / internal.CABLE_NML))
 
-        f90nml.write(deep_update(cable_nml, patch_nml), str(task_dir / internal.CABLE_NML))
+        f90nml.write(deep_update(cable_nml, patch), str(task_dir / internal.CABLE_NML))
 
         return self
 
-    def setup_task(self):
+    def setup_task(self, root_dir=internal.CWD):
         """Does all file manipulations to run cable in the task directory.
 
         These include:
@@ -152,11 +164,15 @@ def setup_task(self):
         into the `runs/site/tasks/<task_name>` directory.
         3. copying the cable executable from the source directory
         4. make appropriate adjustments to namelist files
+        5. apply a branch patch if specified
         """
 
-        self.clean_task() \
-            .fetch_files() \
-            .adjust_namelist_file()
+        self.clean_task(root_dir=root_dir) \
+            .fetch_files(root_dir=root_dir) \
+            .adjust_namelist_file(root_dir=root_dir)
+
+        if self.branch_patch:
+            self.patch_namelist_file(self.branch_patch, root_dir=root_dir)
 
 
 def get_fluxnet_tasks(config: dict, science_config: dict, met_sites: list[str]) -> list[Task]:
@@ -166,6 +182,7 @@ def get_fluxnet_tasks(config: dict, science_config: dict, met_sites: list[str])
         Task(
             branch_id=id,
             branch_name=branch["name"],
+            branch_patch=branch["patch"],
             met_forcing_file=site,
             sci_conf_key=key,
             sci_config=science_config[key]

tests/common.py

@@ -23,12 +23,16 @@ def make_barebones_config() -> dict:
                 "revision": 9000,
                 "trunk": True,
                 "share_branch": False,
+                "patch": {},
             },
             1: {
                 "name": "v3.0-YP-changes",
                 "revision": -1,
                 "trunk": False,
                 "share_branch": False,
+                "patch": {
+                    "cable": {"cable_user": {"ENABLE_SOME_FEATURE": False}}
+                },
             },
         },
     }

tests/test_bench_config.py

@@ -21,6 +21,11 @@ def test_check_config():
     config["realisations"][0].pop("revision")
     check_config(config)
 
+    # Success case: branch configuration with missing patch key
+    config = make_barebones_config()
+    config["realisations"][0].pop("patch")
+    check_config(config)
+
     # Success case: test experiment with site id from the
     # five-site-test is valid
     config = make_barebones_config()
@@ -156,6 +161,12 @@ def test_check_config():
         config["realisations"][1]["share_branch"] = "0"
         check_config(config)
 
+    # Failure case: type of patch key is not a dictionary
+    with pytest.raises(TypeError):
+        config = make_barebones_config()
+        config["realisations"][1]["patch"] = r"cable_user%ENABLE_SOME_FEATURE = .FALSE."
+        check_config(config)
+
 
 def test_read_config():
     """Tests for `read_config()`."""
@@ -185,6 +196,20 @@ def test_read_config():
     assert config != res
     assert res["realisations"][0]["revision"] == -1
 
+    # Success case: a specified branch with a missing patch dictionary
+    # should return a config with patch set to its default value
+    config = make_barebones_config()
+    config["realisations"][0].pop("patch")
+    filename = "config-barebones.yaml"
+
+    with open(filename, "w", encoding="utf-8") as file:
+        yaml.dump(config, file)
+
+    res = read_config(filename)
+    os.remove(filename)
+    assert config != res
+    assert res["realisations"][0]["patch"] == {}
+
 
 def test_check_science_config():
     """Tests for `check_science_config()`."""

tests/test_benchtree.py

@@ -41,14 +41,14 @@ def test_setup_directory_tree():
     sci_id_a, sci_id_b = get_science_config_id(key_a), get_science_config_id(key_b)
 
     tasks = [
-        Task(branch_id_a, branch_name_a, met_site_a, key_a, science_config[key_a]),
-        Task(branch_id_a, branch_name_a, met_site_a, key_b, science_config[key_b]),
-        Task(branch_id_a, branch_name_a, met_site_b, key_a, science_config[key_a]),
-        Task(branch_id_a, branch_name_a, met_site_b, key_b, science_config[key_b]),
-        Task(branch_id_b, branch_name_b, met_site_a, key_a, science_config[key_a]),
-        Task(branch_id_b, branch_name_b, met_site_a, key_b, science_config[key_b]),
-        Task(branch_id_b, branch_name_b, met_site_b, key_a, science_config[key_a]),
-        Task(branch_id_b, branch_name_b, met_site_b, key_b, science_config[key_b]),
+        Task(branch_id_a, branch_name_a, {}, met_site_a, key_a, science_config[key_a]),
+        Task(branch_id_a, branch_name_a, {}, met_site_a, key_b, science_config[key_b]),
+        Task(branch_id_a, branch_name_a, {}, met_site_b, key_a, science_config[key_a]),
+        Task(branch_id_a, branch_name_a, {}, met_site_b, key_b, science_config[key_b]),
+        Task(branch_id_b, branch_name_b, {}, met_site_a, key_a, science_config[key_a]),
+        Task(branch_id_b, branch_name_b, {}, met_site_a, key_b, science_config[key_b]),
+        Task(branch_id_b, branch_name_b, {}, met_site_b, key_a, science_config[key_a]),
+        Task(branch_id_b, branch_name_b, {}, met_site_b, key_b, science_config[key_b]),
     ]
 
     setup_fluxnet_directory_tree(fluxnet_tasks=tasks, root_dir=TMP_DIR)
@@ -82,15 +82,14 @@ def test_clean_directory_tree():
     key_a, key_b = science_config
 
     tasks = [
-        Task(branch_id_a, branch_name_a, met_site_a, key_a, science_config[key_a]),
-        Task(branch_id_a, branch_name_a, met_site_a, key_b, science_config[key_b]),
-        Task(branch_id_a, branch_name_a, met_site_b, key_a, science_config[key_a]),
-        Task(branch_id_a, branch_name_a, met_site_b, key_b, science_config[key_b]),
-        Task(branch_id_b, branch_name_b, met_site_a, key_a, science_config[key_a]),
-        Task(branch_id_b, branch_name_b, met_site_a, key_b, science_config[key_b]),
-        Task(branch_id_b, branch_name_b, met_site_b, key_a, science_config[key_a]),
-        Task(branch_id_b, branch_name_b, met_site_b, key_b, science_config[key_b]),
-
+        Task(branch_id_a, branch_name_a, {}, met_site_a, key_a, science_config[key_a]),
+        Task(branch_id_a, branch_name_a, {}, met_site_a, key_b, science_config[key_b]),
+        Task(branch_id_a, branch_name_a, {}, met_site_b, key_a, science_config[key_a]),
+        Task(branch_id_a, branch_name_a, {}, met_site_b, key_b, science_config[key_b]),
+        Task(branch_id_b, branch_name_b, {}, met_site_a, key_a, science_config[key_a]),
+        Task(branch_id_b, branch_name_b, {}, met_site_a, key_b, science_config[key_b]),
+        Task(branch_id_b, branch_name_b, {}, met_site_b, key_a, science_config[key_a]),
+        Task(branch_id_b, branch_name_b, {}, met_site_b, key_b, science_config[key_b]),
     ]
 
     setup_fluxnet_directory_tree(fluxnet_tasks=tasks, root_dir=TMP_DIR)

tests/test_task.py

@@ -34,6 +34,19 @@ def touch(path):
         os.utime(path, None)
 
 
+def setup_mock_task() -> Task:
+    """Returns a mock `Task` instance."""
+    task = Task(
+        branch_id=1,
+        branch_name="test-branch",
+        branch_patch={"cable": {"some_branch_specific_setting": True}},
+        met_forcing_file="forcing-file.nc",
+        sci_conf_key="sci0",
+        sci_config={"some_setting": True}
+    )
+    return task
+
+
 def setup_mock_namelists_directory():
     """Setup a mock namelists directory in TMP_DIR."""
     Path(TMP_DIR, internal.NAMELIST_DIR).mkdir()
@@ -74,29 +87,29 @@ def do_mock_run(task: Task):
 def test_get_task_name():
     """Tests for `get_task_name()`."""
     # Success case: check task name convention
-    task = Task(1, "test-branch", "forcing-file.nc", "sci0", {"some_setting": True})
+    task = setup_mock_task()
     assert task.get_task_name() == "forcing-file_R1_S0"
 
 
 def test_get_log_filename():
     """Tests for `get_log_filename()`."""
     # Success case: check log file name convention
-    task = Task(1, "test-branch", "forcing-file.nc", "sci0", {"some_setting": True})
+    task = setup_mock_task()
     assert task.get_log_filename() == "forcing-file_R1_S0_log.txt"
 
 
 def test_get_output_filename():
     """Tests for `get_output_filename()`."""
     # Success case: check output file name convention
-    task = Task(1, "test-branch", "forcing-file.nc", "sci0", {"some_setting": True})
+    task = setup_mock_task()
     assert task.get_output_filename() == "forcing-file_R1_S0_out.nc"
 
 
 def test_fetch_files():
     """Tests for `fetch_files()`."""
 
     # Success case: fetch files required to run CABLE
-    task = Task(1, "test-branch", "forcing-file.nc", "sci0", {"some_setting": True})
+    task = setup_mock_task()
 
     setup_mock_namelists_directory()
     setup_fluxnet_directory_tree([task], root_dir=TMP_DIR)
@@ -118,7 +131,7 @@ def test_clean_task():
     """Tests for `clean_task()`."""
 
     # Success case: fetch then clean files
-    task = Task(1, "test-branch", "forcing-file.nc", "sci0", {"some_setting": True})
+    task = setup_mock_task()
 
     setup_mock_namelists_directory()
     setup_fluxnet_directory_tree([task], root_dir=TMP_DIR)
@@ -146,7 +159,7 @@ def test_adjust_namelist_file():
     """Tests for `adjust_namelist_file()`."""
 
     # Success case: adjust cable namelist file
-    task = Task(1, "test-branch", "forcing-file.nc", "sci0", {"some_setting": True})
+    task = setup_mock_task()
     task_dir = Path(TMP_DIR, internal.SITE_TASKS_DIR, task.get_task_name())
 
     setup_fluxnet_directory_tree([task], root_dir=TMP_DIR)
@@ -189,3 +202,21 @@ def test_adjust_namelist_file():
 
     assert res_nml['cable']['filename']['foo'] == 123, "assert existing derived types are preserved"
     assert res_nml['cable']['bar'] == 123, "assert existing top-level parameters are preserved"
+
+
+def test_setup_task():
+    """Tests for `setup_task()`."""
+
+    # Success case: test branch specific settings are patched into task namelist file
+    task = setup_mock_task()
+    task_dir = Path(TMP_DIR, internal.SITE_TASKS_DIR, task.get_task_name())
+
+    setup_mock_namelists_directory()
+    setup_fluxnet_directory_tree([task], root_dir=TMP_DIR)
+    do_mock_checkout_and_build()
+
+    task.setup_task(root_dir=TMP_DIR)
+
+    res_nml = f90nml.read(str(task_dir / internal.CABLE_NML))
+
+    assert res_nml['cable']['some_branch_specific_setting'] is True