From 71191faef9ea8f524fe1cb24f4f52a252cd2c943 Mon Sep 17 00:00:00 2001
From: "Anthony D. Blaom" <anthony.blaom@gmail.com>
Date: Thu, 19 May 2022 09:23:57 +1200
Subject: [PATCH 1/7] add test `level`, replacing `load_only` (breaking)

---
 src/attemptors.jl  |  4 +--
 src/test.jl        | 49 ++++++++++++++++++-------------
 test/attemptors.jl |  4 +--
 test/test.jl       | 72 +++++++++++++++++++++++++++++++++++++++-------
 4 files changed, 94 insertions(+), 35 deletions(-)

diff --git a/src/attemptors.jl b/src/attemptors.jl
index 0247bda..b22b7dd 100644
--- a/src/attemptors.jl
+++ b/src/attemptors.jl
@@ -1,5 +1,3 @@
-str(model_metadata) = "$(model_metadata.name) from $(model_metadata.package_name)"
-
 """
     attempt(f, message="")
 
@@ -11,7 +9,7 @@ If `message` is not empty, then it is logged to `Info`, together with
 the second return value ("✓" or "×").
 
 """
-function attempt(f, message="")
+function attempt(f, message)
     ret = try
         (f(), "✓")
     catch ex
diff --git a/src/test.jl b/src/test.jl
index 65c552d..c2c55c1 100644
--- a/src/test.jl
+++ b/src/test.jl
@@ -9,10 +9,8 @@ function next!(p)
     MLJ.ProgressMeter.updateProgress!(p)
 end
 
-
-
 """
-    test(models, data...; verbosity=1, mod=Main, loading_only=false)
+    test(models, data...; mod=Main, level=2, verbosity=1)
 
 Apply a battery of MLJ integration tests to a collection of models,
 using `data` for training. Here `mod` should be the module from which
@@ -30,9 +28,13 @@ using `data` for training. Here `mod` should be the module from which
   interface packages providing the models must be in the current
   environment, but the packages need not be loaded.
 
-Specify `loading_only=true` to only test model loading, as detailed in
-the test called `model_type` below.
+The extent of testing is controlled by `level`:
 
+|`level`          | description                      | tests (full list below) |
+|:----------------|:---------------------------------|:------------------------|
+| 1               | test code loading                | `:model_type`           |
+| 2 (default)     | basic test of model interface    | first four tests        |
+| 3               | comprehensive                    | all applicable tests    |
 
 # Return value
 
@@ -43,18 +45,18 @@ Returns `(failures, summary)` where:
 - `summary`: table summarizing the outcomes of each test, where
   outcomes are indicated as below:
 
-`summary` table entry | interpretation
-----------------------|-----------------
-✓                     | test succesful
-×                     | test unsuccessful
-n/a                   | skipped because not applicable
- -                    | test skipped for some other reason
+| entry | interpretation                     |
+|:------|:-----------------------------------|
+| ✓     | test succesful                     |
+| ×     | test unsuccessful                  |
+| n/a   | skipped because not applicable     |
+| -     | test skipped for some other reason |
 
 # Examples
 
 ## Testing models in a new MLJ model interface implementation
 
-The following applies the integration tests to a model type
+The following tests the model interface implemented by some model type
 `MyClassifier`, as might appear in tests for a package providing that
 type:
 
@@ -68,10 +70,10 @@ failures, summary = MLJTest.test([MyClassifier, ], X, y, verbosity=1, mod=@__MOD
 
 ## Testing models after filtering models in the registry
 
-The following applies integration tests to all regressors provided by
-the package GLM.jl that are also in the MLJ Model Registry. Since
-GLM.jl models are provided through the interface package
-`MLJGLMInterface`, this must be in the current environment:
+The following applies comprehensive integration tests to all
+regressors provided by the package GLM.jl appearing in the MLJ Model
+Registry. Since GLM.jl models are provided through the interface
+package `MLJGLMInterface`, this must be in the current environment:
 
 ```
 Pkg.add("MLJGLMInterface")
@@ -81,11 +83,17 @@ X, y = MLJTest.MLJ.make_regression();
 regressors = MLJTest.MLJ.models(matching(X, y)) do m
     m.package_name == "GLM"
 end
-failures, summary = MLJTest.test(regressors, X, y, verbosity=1, mod=@__MODULE__)
+failures, summary = MLJTest.test(
+    regressors, 
+    X, 
+    y, 
+    verbosity=1, 
+    mod=@__MODULE__,
+    level=3)
 summary |> DataFrame
 ```
 
-# List of tests applied
+# List of tests
 
 Tests are applied in sequence. When a test fails, subsequent tests for
 that model are skipped. The following are applied to all models:
@@ -121,7 +129,7 @@ These additional tests are applied to `Supervised` models:
   but first wrap as an `IteratedModel`.
 
 """
-function test(model_proxies, data...; verbosity=1, mod=Main, load_only=false)
+function test(model_proxies, data...; verbosity=1, mod=Main, level=2)
 
     nproxies = length(model_proxies)
 
@@ -200,7 +208,7 @@ function test(model_proxies, data...; verbosity=1, mod=Main, load_only=false)
         row = update(row, i, :model_type, model_type, outcome)
         outcome == "×" && continue
 
-        load_only && continue
+        level > 1 || continue
 
         # model_instance:
         model_instance, outcome =
@@ -225,6 +233,7 @@ function test(model_proxies, data...; verbosity=1, mod=Main, load_only=false)
             row = update(row, i, :operations, operations, operations)
         end
 
+        level > 2 || continue
         model_instance isa Supervised || continue
 
         # supervised tests:
diff --git a/test/attemptors.jl b/test/attemptors.jl
index dc8a554..315ea9d 100644
--- a/test/attemptors.jl
+++ b/test/attemptors.jl
@@ -3,9 +3,9 @@
     bad() = throw(e)
     good() = 42
 
-    @test (@test_logs MLJTest.attempt(bad)) == (e, "×")
+    @test (@test_logs MLJTest.attempt(bad, "")) == (e, "×")
     @test (@test_logs (:info, "look ×") MLJTest.attempt(bad, "look "))  == (e, "×")
-    @test (@test_logs MLJTest.attempt(good)) == (42, "✓")
+    @test (@test_logs MLJTest.attempt(good, "")) == (42, "✓")
     @test (@test_logs (:info, "look ✓") MLJTest.attempt(good, "look "))  == (42, "✓")
 end
 
diff --git a/test/test.jl b/test/test.jl
index 7896591..c7eac45 100644
--- a/test/test.jl
+++ b/test/test.jl
@@ -37,7 +37,14 @@ expected_summary2 = (
     y = coerce(y0, OrderedFactor);
 
     fails, summary  =
-        @test_logs MLJTest.test(classifiers, X, y; mod=@__MODULE__, verbosity=0)
+        @test_logs MLJTest.test(
+            classifiers,
+            X,
+            y;
+            mod=@__MODULE__,
+            level=3,
+            verbosity=0
+        )
     @test isempty(fails)
     @test summary[1] == expected_summary1
     @test summary[2] == expected_summary2
@@ -50,7 +57,14 @@ end
     y = coerce(y0, OrderedFactor);
 
     fails, summary  =
-        @test_logs MLJTest.test(classifiers, X, y; mod=@__MODULE__, verbosity=0)
+        @test_logs MLJTest.test(
+            classifiers,
+            X,
+            y;
+            mod=@__MODULE__,
+            level=3,
+            verbosity=0
+        )
     @test isempty(fails)
     @test summary[1] == expected_summary1
     @test summary[2] == expected_summary2
@@ -61,7 +75,14 @@ end
     fails, summary = @test_logs(
         (:error, r""),
         match_mode=:any,
-        MLJTest.test(classifiers, X, y; mod=@__MODULE__, verbosity=0)
+        MLJTest.test(
+            classifiers,
+            X,
+            y;
+            mod=@__MODULE__,
+            level=3,
+            verbosity=0
+        )
     )
 
     @test length(fails) === 2
@@ -122,7 +143,7 @@ y = coerce(y0, OrderedFactor);
         X,
         y;
         mod=@__MODULE__,
-        load_only=true,
+        level=1,
         verbosity=1);
 
     # verbosity high:
@@ -141,19 +162,20 @@ y = coerce(y0, OrderedFactor);
             X,
             y;
             mod=@__MODULE__,
+            level=3,
             verbosity=2)
     )
-end 
-
-@testset "load_only=true" begin
+end
 
+@testset "level" begin
+    # level=1:
     fails, summary  =
         @test_logs MLJTest.test(
             classifiers,
             X,
             y;
             mod=@__MODULE__,
-            load_only=true,
+            level=1,
             verbosity=0)
     @test isempty(fails)
     @test summary[1] == (
@@ -169,13 +191,43 @@ end
         ensemble_prediction = "-",
         iteration_prediction = "-",
     )
-end
 
+    # level=2:
+    fails, summary  =
+        @test_logs MLJTest.test(
+            classifiers,
+            X,
+            y;
+            mod=@__MODULE__,
+            level=2,
+            verbosity=0)
+    @test isempty(fails)
+    @test summary[1] == (
+        name = "ConstantClassifier",
+        package = "MLJModels",
+        model_type = "✓",
+        model_instance = "✓",
+        fitted_machine = "✓",
+        operations = "predict",
+        evaluation = "-",
+        tuned_pipe_evaluation = "-",
+        threshold_prediction = "-",
+        ensemble_prediction = "-",
+        iteration_prediction = "-",
+    )
+end
 
 @testset "iterative model" begin
     X, y = MLJTest.make_dummy();
     fails, summary =
-        MLJTest.test([MLJTest.DummyIterativeModel,], X, y; mod=@__MODULE__, verbosity=0)
+        MLJTest.test(
+            [MLJTest.DummyIterativeModel,],
+            X,
+            y;
+            mod=@__MODULE__,
+            level=3,
+            verbosity=0
+        )
     @test isempty(fails)
     @test summary[1] == (
         name = "DummyIterativeModel",

From 626418689750e80cfe0320c298d307b6916184f4 Mon Sep 17 00:00:00 2001
From: "Anthony D. Blaom" <anthony.blaom@gmail.com>
Date: Thu, 19 May 2022 09:25:43 +1200
Subject: [PATCH 2/7] fix doc-string

---
 src/attemptors.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/attemptors.jl b/src/attemptors.jl
index b22b7dd..f55052e 100644
--- a/src/attemptors.jl
+++ b/src/attemptors.jl
@@ -1,5 +1,5 @@
 """
-    attempt(f, message="")
+    attempt(f, message)
 
 Return `(f(), "✓") if `f()` executes without throwing an
 exception. Otherwise, return `(ex, "×"), where `ex` is the exception

From 6d68c6978bc1c99be7041fbb5abc762405d881a2 Mon Sep 17 00:00:00 2001
From: "Anthony D. Blaom" <anthony.blaom@gmail.com>
Date: Thu, 19 May 2022 10:11:38 +1200
Subject: [PATCH 3/7] add option `throw` to throw first exception encountered

---
 src/attemptors.jl  | 46 ++++++++++++++++++++++++----------------------
 src/test.jl        | 33 ++++++++++++++++++++++-----------
 test/attemptors.jl |  1 +
 test/test.jl       | 16 ++++++++++++++++
 4 files changed, 63 insertions(+), 33 deletions(-)

diff --git a/src/attemptors.jl b/src/attemptors.jl
index f55052e..cbe5eeb 100644
--- a/src/attemptors.jl
+++ b/src/attemptors.jl
@@ -1,18 +1,20 @@
 """
-    attempt(f, message)
+    attempt(f, message; throw=false)
 
 Return `(f(), "✓") if `f()` executes without throwing an
 exception. Otherwise, return `(ex, "×"), where `ex` is the exception
-thrown.
+caught. Only truly throw the exception if `throw=true`. 
 
 If `message` is not empty, then it is logged to `Info`, together with
 the second return value ("✓" or "×").
 
+
 """
-function attempt(f, message)
+function attempt(f, message; throw=false)
     ret = try
         (f(), "✓")
     catch ex
+        throw && Base.throw(ex)
         (ex, "×")
     end
     isempty(message) || @info message*last(ret)
@@ -37,10 +39,10 @@ end
 
 root(load_path) = split(load_path, '.') |> first
 
-function model_type(proxy, mod; verbosity=1)
+function model_type(proxy, mod; throw=false, verbosity=1)
     # check interface package really is in current environment:
     message = "`[:model_type]` Loading model type "
-    model_type, outcome = attempt(finalize(message, verbosity)) do
+    model_type, outcome = attempt(finalize(message, verbosity); throw) do
         load_path = MLJTest.load_path(proxy) # MLJ.load_path(proxy) *****
         load_path_ex = load_path |> Meta.parse
         api_pkg_ex = root(load_path) |> Symbol
@@ -67,31 +69,31 @@ function model_type(proxy, mod; verbosity=1)
         if !isnothing(api_pkg) &&
                api_pkg != "unknown" &&
                contains(model_type.msg, "$api_pkg not found in")
-            throw(model_type)
+            Base.throw(model_type)
         end
     end
 
     return model_type, outcome
 end
 
-function model_instance(model_type; verbosity=1)
+function model_instance(model_type; throw=false, verbosity=1)
     message = "`[:model_instance]` Instantiating default model "
-    attempt(finalize(message, verbosity))  do
+    attempt(finalize(message, verbosity); throw)  do
         model_type()
     end
 end
 
-function fitted_machine(model, data...; verbosity=1)
+function fitted_machine(model, data...; throw=false, verbosity=1)
     message = "`[:fitted_machine]` Fitting machine "
-    attempt(finalize(message, verbosity))  do
+    attempt(finalize(message, verbosity); throw)  do
         mach = machine(model, data...)
         fit!(mach, verbosity=-1)
     end
 end
 
-function operations(fitted_machine, data...; verbosity=1)
+function operations(fitted_machine, data...; throw=false, verbosity=1)
     message = "`[:operations]` Calling `predict`, `transform` and/or `inverse_transform` "
-    attempt(finalize(message, verbosity))  do
+    attempt(finalize(message, verbosity); throw)  do
         operations = String[]
         methods = MLJ.implemented_methods(fitted_machine.model)
         if :predict in methods
@@ -110,10 +112,10 @@ function operations(fitted_machine, data...; verbosity=1)
     end
 end
 
-function threshold_prediction(model, data...; verbosity=1)
+function threshold_prediction(model, data...; throw=false, verbosity=1)
     message = "`[:threshold_predictor]` Calling fit!/predict for threshold predictor "*
         "test) "
-    attempt(finalize(message, verbosity)) do
+    attempt(finalize(message, verbosity); throw) do
         tmodel = BinaryThresholdPredictor(model)
         mach = machine(tmodel, data...)
         fit!(mach, verbosity=0)
@@ -121,9 +123,9 @@ function threshold_prediction(model, data...; verbosity=1)
     end
 end
 
-function evaluation(measure, model, data...; verbosity=1)
+function evaluation(measure, model, data...; throw=false, verbosity=1)
     message = "`[:evaluation]` Evaluating performance "
-    attempt(finalize(message, verbosity)) do
+    attempt(finalize(message, verbosity); throw) do
         evaluate(model, data...;
                  measure=measure,
                  resampling=Holdout(),
@@ -131,9 +133,9 @@ function evaluation(measure, model, data...; verbosity=1)
     end
 end
 
-function tuned_pipe_evaluation(measure, model, data...; verbosity=1)
+function tuned_pipe_evaluation(measure, model, data...; throw=false, verbosity=1)
     message = "`[:tuned_pipe_evaluation]` Evaluating perfomance in a tuned pipeline "
-    attempt(finalize(message, verbosity)) do
+    attempt(finalize(message, verbosity); throw) do
         pipe = identity |> model
         tuned_pipe = TunedModel(models=[pipe,],
                                 measure=measure)
@@ -143,8 +145,8 @@ function tuned_pipe_evaluation(measure, model, data...; verbosity=1)
     end
 end
 
-function ensemble_prediction(model, data...; verbosity=1)
-    attempt(finalize("`[:ensemble_prediction]` Ensembling ", verbosity)) do
+function ensemble_prediction(model, data...; throw=false, verbosity=1)
+    attempt(finalize("`[:ensemble_prediction]` Ensembling ", verbosity); throw) do
         imodel = EnsembleModel(model=model,
                                n=2)
         mach = machine(imodel, data...)
@@ -153,9 +155,9 @@ function ensemble_prediction(model, data...; verbosity=1)
     end
 end
 
-function iteration_prediction(measure, model, data...; verbosity=1)
+function iteration_prediction(measure, model, data...; throw=false, verbosity=1)
     message =  "`[:iteration_prediction]` Iterating with controls "
-    attempt(finalize(message, verbosity)) do
+    attempt(finalize(message, verbosity); throw) do
         imodel = IteratedModel(model=model,
                                measure=measure,
                                controls=[Step(1),
diff --git a/src/test.jl b/src/test.jl
index c2c55c1..143f941 100644
--- a/src/test.jl
+++ b/src/test.jl
@@ -10,7 +10,7 @@ function next!(p)
 end
 
 """
-    test(models, data...; mod=Main, level=2, verbosity=1)
+    test(models, data...; mod=Main, level=2, throw=false, verbosity=1)
 
 Apply a battery of MLJ integration tests to a collection of models,
 using `data` for training. Here `mod` should be the module from which
@@ -36,6 +36,11 @@ The extent of testing is controlled by `level`:
 | 2 (default)     | basic test of model interface    | first four tests        |
 | 3               | comprehensive                    | all applicable tests    |
 
+By default, exceptions caught in tests are not thrown. If
+`throw=true`, testing will terminate at the first execption
+encountered, after throwing that exception (useful to obtain stack
+traces).
+
 # Return value
 
 Returns `(failures, summary)` where:
@@ -129,7 +134,7 @@ These additional tests are applied to `Supervised` models:
   but first wrap as an `IteratedModel`.
 
 """
-function test(model_proxies, data...; verbosity=1, mod=Main, level=2)
+function test(model_proxies, data...; mod=Main, level=2, throw=false, verbosity=1,)
 
     nproxies = length(model_proxies)
 
@@ -204,7 +209,7 @@ function test(model_proxies, data...; verbosity=1, mod=Main, level=2)
         row = merge(row0, (; name, package))
 
         # model_type:
-        model_type, outcome = MLJTest.model_type(model_proxy, mod; verbosity)
+        model_type, outcome = MLJTest.model_type(model_proxy, mod; throw, verbosity)
         row = update(row, i, :model_type, model_type, outcome)
         outcome == "×" && continue
 
@@ -212,19 +217,19 @@ function test(model_proxies, data...; verbosity=1, mod=Main, level=2)
 
         # model_instance:
         model_instance, outcome =
-            MLJTest.model_instance(model_type; verbosity)
+            MLJTest.model_instance(model_type; throw, verbosity)
         row = update(row, i, :model_instance, model_instance, outcome)
         outcome == "×" && continue
 
         # fitted_machine:
         fitted_machine, outcome =
-            MLJTest.fitted_machine(model_instance, data...; verbosity)
+            MLJTest.fitted_machine(model_instance, data...; throw, verbosity)
         row = update(row, i, :fitted_machine, fitted_machine, outcome)
         outcome == "×" && continue
 
         # operations:
         operations, outcome =
-            MLJTest.operations(fitted_machine, data...; verbosity)
+            MLJTest.operations(fitted_machine, data...; throw, verbosity)
         # special treatment to get list of operations in `summary`:
         if operations == "×"
             row = update(row, i, :operations, operations, outcome)
@@ -245,7 +250,7 @@ function test(model_proxies, data...; verbosity=1, mod=Main, level=2)
             scitype(data[2]) <: AbstractVector{<:Finite{2}}
 
             threshold_prediction, outcome =
-                MLJTest.threshold_prediction(model_instance, data...; verbosity)
+                MLJTest.threshold_prediction(model_instance, data...; throw, verbosity)
             row = update(row, i, :threshold_prediction, threshold_prediction, outcome)
             outcome == "×" && continue
         end
@@ -256,19 +261,25 @@ function test(model_proxies, data...; verbosity=1, mod=Main, level=2)
 
         # evaluation:
         evaluation, outcome =
-            MLJTest.evaluation(measure, model_instance, data...; verbosity)
+            MLJTest.evaluation(measure, model_instance, data...; throw, verbosity)
         row = update(row, i, :evaluation, evaluation, outcome)
         outcome == "×" && continue
 
         # tuned_pipe_evaluation:
         tuned_pipe_evaluation, outcome =
-            MLJTest.tuned_pipe_evaluation(measure, model_instance, data...; verbosity)
+            MLJTest.tuned_pipe_evaluation(
+                measure,
+                model_instance,
+                data...;
+                throw,
+                verbosity
+            )
         row = update(row, i, :tuned_pipe_evaluation, tuned_pipe_evaluation, outcome)
         outcome == "×" && continue
 
         # ensemble_prediction:
         ensemble_prediction, outcome =
-            MLJTest.ensemble_prediction(model_instance, data...; verbosity)
+            MLJTest.ensemble_prediction(model_instance, data...; throw, verbosity)
         row = update(row, i, :ensemble_prediction, ensemble_prediction, outcome)
         outcome == "×" && continue
 
@@ -276,7 +287,7 @@ function test(model_proxies, data...; verbosity=1, mod=Main, level=2)
 
         # iteration prediction:
         iteration_prediction, outcome =
-            MLJTest.iteration_prediction(measure, model_instance, data...; verbosity)
+            MLJTest.iteration_prediction(measure, model_instance, data...; throw, verbosity)
         row = update(row, i, :iteration_prediction, iteration_prediction, outcome)
         outcome == "×" && continue
     end
diff --git a/test/attemptors.jl b/test/attemptors.jl
index 315ea9d..88b73b1 100644
--- a/test/attemptors.jl
+++ b/test/attemptors.jl
@@ -7,6 +7,7 @@
     @test (@test_logs (:info, "look ×") MLJTest.attempt(bad, "look "))  == (e, "×")
     @test (@test_logs MLJTest.attempt(good, "")) == (42, "✓")
     @test (@test_logs (:info, "look ✓") MLJTest.attempt(good, "look "))  == (42, "✓")
+    @test_throws e MLJTest.attempt(bad, ""; throw=true)
 end
 
 @testset "model_type" begin
diff --git a/test/test.jl b/test/test.jl
index c7eac45..20bfb7b 100644
--- a/test/test.jl
+++ b/test/test.jl
@@ -130,6 +130,22 @@ end
         iteration_prediction = "-"
     )
 
+    # throw=true:
+    @test_logs(
+        (:error, r""), match_mode=:any,
+        @test_throws(
+            ErrorException,
+            MLJTest.test(
+                classifiers,
+                X,
+                y;
+                mod=@__MODULE__,
+                level=3,
+                throw=true,
+                verbosity=0
+            )
+        )
+    )
 end
 
 classifiers = [ConstantClassifier,]

From 8f7748cb657a048e4ff522c0199631fbf972063c Mon Sep 17 00:00:00 2001
From: "Anthony D. Blaom" <anthony.blaom@gmail.com>
Date: Thu, 19 May 2022 10:12:50 +1200
Subject: [PATCH 4/7] bump 0.2.0

---
 Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Project.toml b/Project.toml
index ca3d06f..4ec26b9 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,7 +1,7 @@
 name = "MLJTest"
 uuid = "697918b4-fdc1-4f9e-8ff9-929724cee270"
 authors = ["Anthony D. Blaom <anthony.blaom@gmail.com>"]
-version = "0.1.0"
+version = "0.2.0"
 
 [deps]
 MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"

From a8f4a48bec3b3a29904ea42158a9468c31f05ca5 Mon Sep 17 00:00:00 2001
From: "Anthony D. Blaom" <anthony.blaom@gmail.com>
Date: Thu, 19 May 2022 10:15:50 +1200
Subject: [PATCH 5/7] copy examples to README.md

---
 README.md | 44 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 43 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5d93093..933da9b 100644
--- a/README.md
+++ b/README.md
@@ -23,4 +23,46 @@ using the specified training `data`:
 MLJTest.test(models, data...; verbosity=1, mod=Main, loading_only=false) -> failures, summary
 ```
 
-See the method document string for details. 
+For detailed documentation, run `using MLJTest; @doc MLJTest.test`.
+
+
+# Examples
+
+## Testing models in a new MLJ model interface implementation
+
+The following tests the model interface implemented by some model type
+`MyClassifier`, as might appear in tests for a package providing that
+type:
+
+```
+import MLJTest
+using Test
+X, y = MLJTest.MLJ.make_blobs()
+failures, summary = MLJTest.test([MyClassifier, ], X, y, verbosity=1, mod=@__MODULE__)
+@test isempty(failures)
+```
+
+## Testing models after filtering models in the registry
+
+The following applies comprehensive integration tests to all
+regressors provided by the package GLM.jl appearing in the MLJ Model
+Registry. Since GLM.jl models are provided through the interface
+package `MLJGLMInterface`, this must be in the current environment:
+
+```
+Pkg.add("MLJGLMInterface")
+import MLJBase, MLJTest
+using DataFrames # to view summary
+X, y = MLJTest.MLJ.make_regression();
+regressors = MLJTest.MLJ.models(matching(X, y)) do m
+    m.package_name == "GLM"
+end
+failures, summary = MLJTest.test(
+    regressors, 
+    X, 
+    y, 
+    verbosity=1, 
+    mod=@__MODULE__,
+    level=3)
+summary |> DataFrame
+```

From 18027532743701e8ee7e4d01880e3ba1a94954b0 Mon Sep 17 00:00:00 2001
From: "Anthony D. Blaom" <anthony.blaom@gmail.com>
Date: Thu, 19 May 2022 10:19:22 +1200
Subject: [PATCH 6/7] fix readme

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 933da9b..f87cc73 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,8 @@ This package provides a single method for testing a collection of
 using the specified training `data`:
 
 ```
-MLJTest.test(models, data...; verbosity=1, mod=Main, loading_only=false) -> failures, summary
+MLJTest.test(models, data...; mod=Main, level=2, throw=false, verbosity=1) 
+    -> failures, summary
 ```
 
 For detailed documentation, run `using MLJTest; @doc MLJTest.test`.

From 6073494bb7831b32ef311db9c0c451a357b99329 Mon Sep 17 00:00:00 2001
From: "Anthony D. Blaom" <anthony.blaom@gmail.com>
Date: Thu, 19 May 2022 10:20:20 +1200
Subject: [PATCH 7/7] doc fixes

---
 README.md   | 6 +++---
 src/test.jl | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index f87cc73..f5aca15 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ This package provides a single method for testing a collection of
 `models` (types or named tuples with keys `:name` and `:package_name`)
 using the specified training `data`:
 
-```
+```julia
 MLJTest.test(models, data...; mod=Main, level=2, throw=false, verbosity=1) 
     -> failures, summary
 ```
@@ -35,7 +35,7 @@ The following tests the model interface implemented by some model type
 `MyClassifier`, as might appear in tests for a package providing that
 type:
 
-```
+```julia
 import MLJTest
 using Test
 X, y = MLJTest.MLJ.make_blobs()
@@ -50,7 +50,7 @@ regressors provided by the package GLM.jl appearing in the MLJ Model
 Registry. Since GLM.jl models are provided through the interface
 package `MLJGLMInterface`, this must be in the current environment:
 
-```
+```julia
 Pkg.add("MLJGLMInterface")
 import MLJBase, MLJTest
 using DataFrames # to view summary
diff --git a/src/test.jl b/src/test.jl
index 143f941..7e58750 100644
--- a/src/test.jl
+++ b/src/test.jl
@@ -65,7 +65,7 @@ The following tests the model interface implemented by some model type
 `MyClassifier`, as might appear in tests for a package providing that
 type:
 
-```
+```julia
 import MLJTest
 using Test
 X, y = MLJTest.MLJ.make_blobs()
@@ -80,7 +80,7 @@ regressors provided by the package GLM.jl appearing in the MLJ Model
 Registry. Since GLM.jl models are provided through the interface
 package `MLJGLMInterface`, this must be in the current environment:
 
-```
+```julia
 Pkg.add("MLJGLMInterface")
 import MLJBase, MLJTest
 using DataFrames # to view summary