test: autotest framework fixes

* remove return statement causing checks to be skipped
* standardize check_output(idx, test) function signature
* consolidate framework logging
* rename ex to cases in test scripts
* use context manager to open files
* run black/isort

autotest/common_regression.py

@@ -423,7 +423,7 @@ def setup_mf6_comparison(
                     break
 
 
-def get_mf6_files(namefile):
+def get_mf6_files(namefile, verbose=False):
     """Get all MODFLOW 6 input and output files in this simulation.
 
     Parameters
@@ -500,6 +500,14 @@ def get_mf6_files(namefile):
         if len(flist) < 1:
             break
 
+    if verbose:
+        from pprint import pprint
+
+        print(f"Found input files for {namefile}:")
+        pprint(pkg_files)
+        print(f"Expecting output files for {namefile}:")
+        pprint(out_files)
+
     return pkg_files, out_files
 
 

autotest/framework.py

@@ -13,8 +13,13 @@
 from modflow_devtools.executables import Executables
 from modflow_devtools.misc import get_ostag, is_in_ci
 
-from common_regression import (get_mf6_comparison, get_mf6_files,
-                               get_namefiles, setup_mf6, setup_mf6_comparison)
+from common_regression import (
+    get_mf6_comparison,
+    get_mf6_files,
+    get_namefiles,
+    setup_mf6,
+    setup_mf6_comparison,
+)
 
 DNODATA = 3.0e30
 EXTS = {
@@ -787,11 +792,12 @@ def run(self):
 
         """
 
-        # build/store models andd simulations and write input files
+        # build/store models and simulations and write input files
         if self.build:
             sims = self.build(self)
             sims = [sims] if not isinstance(sims, Iterable) else sims
             sims = [sim for sim in sims if sim]
+            # todo remove assert if/when arbitrary # of comparison models supported
             assert len(sims) <= 2, "expected at most 2 simulations/models"
             self.sims = {
                 (
@@ -801,11 +807,13 @@ def run(self):
                 ): sim
                 for sim in sims
             }
-            write_input(*sims)
+            write_input(*sims, verbose=self.verbose)
 
         # run main model(s) and get expected output files
         assert self.run_main_model(), "main model(s) failed"
-        _, self.outp = get_mf6_files(self.workspace / "mfsim.nam")
+        _, self.outp = get_mf6_files(
+            self.workspace / "mfsim.nam", self.verbose
+        )
 
         # setup and run comparison model(s), if enabled
         if self.compare:
@@ -826,55 +834,71 @@ def run(self):
 
             # try to autodetect comparison type if enabled
             if self.compare == "auto":
+                if self.verbose:
+                    print("Auto-detecting comparison type")
                 self.compare = get_mf6_comparison(self.workspace)
-            if not self.compare:
-                warn("Could not detect comparison type, aborting comparison")
-                return
-
-            # copy reference model files if mf6 regression
-            if self.compare == "mf6_regression":
-                cmp_path = self.workspace / self.compare
-                if os.path.isdir(cmp_path):
-                    shutil.rmtree(cmp_path)
-                shutil.copytree(self.workspace, cmp_path)
-
-            # run comparison model, don't compare results
-            run_only = self.compare == "run_only"
-
-            # todo: don't hardcode workspace / assume agreement with test case
-            # simulation workspace, store/access sim/model workspaces directly
-            workspace = (
-                self.workspace / "mf6"
-                if run_only
-                else self.workspace / self.compare
-            )
+            if self.compare:
+                if self.verbose:
+                    print(f"Running comparison type: {self.compare}")
+
+                # copy reference model files if mf6 regression
+                if self.compare == "mf6_regression":
+                    cmp_path = self.workspace / self.compare
+                    if os.path.isdir(cmp_path):
+                        if self.verbose:
+                            print(f"Cleaning {cmp_path}")
+                        shutil.rmtree(cmp_path)
+                    if self.verbose:
+                        print(
+                            f"Copying reference model files from {self.workspace} to {cmp_path}"
+                        )
+                    shutil.copytree(self.workspace, cmp_path)
 
-            # look up the target executable, can be
-            #   - mf2005
-            #   - mfnwt
-            #   - mfusg
-            #   - mflgr
-            #   - libmf6
-            #   - mf6
-            #   - mf6_regression
-            exe = self.targets.get(
-                self.compare.lower().replace(".cmp", ""),
-                self.targets.mf6,
-            )
+                # run comparison model, don't compare results
+                run_only = self.compare == "run_only"
 
-            # run comparison model
-            assert self.run_comparison_model(
-                workspace=workspace,
-                exe=exe,
-            ), "comparison model(s) failed"
+                # todo: don't hardcode workspace / assume agreement with test case
+                # simulation workspace, store/access sim/model workspaces directly
+                workspace = (
+                    self.workspace / "mf6"
+                    if run_only
+                    else self.workspace / self.compare
+                )
 
-            # compare model results, if enabled
-            if not run_only:
-                # if mf6 or mf6 regression test, get output files
-                if "mf6" in self.compare:
-                    _, self.coutp = get_mf6_files(self.workspace / "mfsim.nam")
-                self.compare_output(self.compare)
+                # look up the target executable, can be
+                #   - mf2005
+                #   - mfnwt
+                #   - mfusg
+                #   - mflgr
+                #   - libmf6
+                #   - mf6
+                #   - mf6_regression
+                exe = self.targets.get(
+                    self.compare.lower().replace(".cmp", ""),
+                    self.targets.mf6,
+                )
+
+                # run comparison model
+                assert self.run_comparison_model(
+                    workspace=workspace,
+                    exe=exe,
+                ), "comparison model(s) failed"
+
+                # compare model results, if enabled
+                if not run_only:
+                    # if mf6 or mf6 regression test, get output files
+                    if "mf6" in self.compare:
+                        _, self.coutp = get_mf6_files(
+                            self.workspace / "mfsim.nam", self.verbose
+                        )
+                    if self.verbose:
+                        print("Comparing model outputs")
+                    self.compare_output(self.compare)
+            else:
+                warn("Could not detect comparison type, aborting comparison")
 
         # check results, if enabled
         if self.check:
+            if self.verbose:
+                print("Checking model outputs against expectation")
             self.check(self)

autotest/test_gwf_ats01.py

@@ -7,9 +7,10 @@
 import flopy
 import numpy as np
 import pytest
+
 from framework import TestFramework
 
-ex = ["gwf_ats01a"]
+cases = ["gwf_ats01a"]
 nlay, nrow, ncol = 1, 1, 2
 
 # set dt0, dtmin, dtmax, dtadj, dtfailadj
@@ -39,7 +40,7 @@ def build_models(idx, test):
     for id in range(nper):
         tdis_rc.append((perlen[id], nstp[id], tsmult[id]))
 
-    name = ex[idx]
+    name = cases[idx]
 
     # build MODFLOW 6 files
     ws = test.workspace
@@ -173,9 +174,7 @@ def build_models(idx, test):
     return sim, None
 
 
-def check_output(test):
-    print("evaluating flow...")
-
+def check_output(idx, test):
     # This will fail if budget numbers cannot be read
     fpth = os.path.join(test.workspace, f"{test.name}.lst")
     mflist = flopy.utils.Mf6ListBudget(fpth)
@@ -205,14 +204,14 @@ def check_output(test):
 
 @pytest.mark.parametrize(
     "idx, name",
-    list(enumerate(ex)),
+    list(enumerate(cases)),
 )
 def test_mf6model(idx, name, function_tmpdir, targets):
     test = TestFramework(
         name=name,
         workspace=function_tmpdir,
         build=lambda t: build_models(idx, t),
-        check=check_output,
+        check=lambda t: check_output(idx, t),
         targets=targets,
     )
     test.run()

autotest/test_gwf_ats02.py

@@ -8,9 +8,10 @@
 import flopy
 import numpy as np
 import pytest
+
 from framework import TestFramework
 
-ex = ["gwf_ats02a"]
+cases = ["gwf_ats02a"]
 nlay, nrow, ncol = 5, 1, 1
 botm = [80.0, 60.0, 40.0, 20.0, 0.0]
 
@@ -40,7 +41,7 @@ def build_models(idx, test):
     for id in range(nper):
         tdis_rc.append((perlen[id], nstp[id], tsmult[id]))
 
-    name = ex[idx]
+    name = cases[idx]
 
     # build MODFLOW 6 files
     ws = test.workspace
@@ -204,9 +205,7 @@ def make_plot(test):
     plt.show()
 
 
-def check_output(test):
-    print("evaluating flow...")
-
+def check_output(idx, test):
     # This will fail if budget numbers cannot be read
     fpth = os.path.join(test.workspace, f"{test.name}.lst")
     mflist = flopy.utils.Mf6ListBudget(fpth)
@@ -231,14 +230,14 @@ def check_output(test):
 
 @pytest.mark.parametrize(
     "idx, name",
-    list(enumerate(ex)),
+    list(enumerate(cases)),
 )
 def test_mf6model(idx, name, function_tmpdir, targets):
     test = TestFramework(
         name=name,
         workspace=function_tmpdir,
         build=lambda t: build_models(idx, t),
-        check=check_output,
+        check=lambda t: check_output(idx, t),
         targets=targets,
     )
     test.run()

autotest/test_gwf_ats03.py

@@ -14,9 +14,10 @@
 import flopy
 import numpy as np
 import pytest
+
 from framework import TestFramework
 
-ex = ["gwf_ats03a"]
+cases = ["gwf_ats03a"]
 nlay, nrow, ncol = 1, 1, 10
 
 
@@ -39,7 +40,7 @@ def build_models(idx, test):
     for id in range(nper):
         tdis_rc.append((perlen[id], nstp[id], tsmult[id]))
 
-    name = ex[idx]
+    name = cases[idx]
 
     # build MODFLOW 6 files
     ws = test.workspace
@@ -189,9 +190,7 @@ def build_models(idx, test):
     return sim, None
 
 
-def check_output(test):
-    print("evaluating flow...")
-
+def check_output(idx, test):
     # ensure obs2 (a constant head time series) drops linearly from 100 to 50
     fpth = os.path.join(test.workspace, test.name + ".obs.csv")
     try:
@@ -207,14 +206,14 @@ def check_output(test):
 
 @pytest.mark.parametrize(
     "idx, name",
-    list(enumerate(ex)),
+    list(enumerate(cases)),
 )
 def test_mf6model(idx, name, function_tmpdir, targets):
     test = TestFramework(
         name=name,
         workspace=function_tmpdir,
         build=lambda t: build_models(idx, t),
-        check=check_output,
+        check=lambda t: check_output(idx, t),
         targets=targets,
     )
     test.run()

autotest/test_gwf_ats_lak01.py

@@ -9,9 +9,10 @@
 import flopy
 import numpy as np
 import pytest
+
 from framework import TestFramework
 
-ex = ["gwf_ats_lak_01a"]
+cases = ["gwf_ats_lak_01a"]
 gwf = None
 
 
@@ -49,7 +50,7 @@ def build_models(idx, test):
     nouter, ninner = 250, 300
     hclose, rclose, relax = 1e-8, 1e-6, 0.97
 
-    name = ex[idx]
+    name = cases[idx]
 
     # build MODFLOW 6 files
     ws = test.workspace
@@ -289,9 +290,7 @@ def get_kij_from_node(node, nrow, ncol):
     return k, i, j
 
 
-def check_output(test):
-    print("evaluating results...")
-
+def check_output(idx, test):
     # calculate volume of water and make sure it is conserved
     fname = test.name + ".lak.bin"
     fname = os.path.join(test.workspace, fname)
@@ -412,14 +411,14 @@ def check_output(test):
 @pytest.mark.slow
 @pytest.mark.parametrize(
     "idx, name",
-    list(enumerate(ex)),
+    list(enumerate(cases)),
 )
 def test_mf6model(idx, name, function_tmpdir, targets):
     test = TestFramework(
         name=name,
         workspace=function_tmpdir,
         build=lambda t: build_models(idx, t),
-        check=check_output,
+        check=lambda t: check_output(idx, t),
         targets=targets,
     )
     test.run()