From caf053070595aff4319f20646a29a976e5a555d6 Mon Sep 17 00:00:00 2001 From: Bastian Krause Date: Sat, 4 Jan 2025 13:51:03 +0100 Subject: [PATCH] python: allow adding parameter names to parametrized test IDs By default, only the parameter's values make it into parametrized test IDs. The parameter names don't. Since parameter values do not always speak for themselves, the test function + test ID are often not descriptive/expressive. Allowing parameter name=value pairs in the test ID optionally to get an idea what parameters a test gets passed is beneficial. So add a kwarg `id_names` to @pytest.mark.parametrize() / pytest.Metafunc.parametrize(). It defaults to `False` to keep the auto-generated ID as before. If set to `True`, the argument parameter=value pairs in the auto-generated test IDs are enabled. Calling parametrize() with `ids` and `id_names=True` is considered an error. Auto-generated test ID with `id_names=False` (default behavior as before): test_something[100-10-True-False-True] Test ID with `id_names=True`: test_something[speed_down=100-speed_up=10-foo=True-bar=False-baz=True] Signed-off-by: Bastian Krause --- AUTHORS | 1 + changelog/13055.feature.rst | 1 + doc/en/example/parametrize.rst | 43 +++++++++++++++++++++------------- src/_pytest/python.py | 35 +++++++++++++++++++++------ testing/python/metafunc.py | 30 ++++++++++++++++++++---- 5 files changed, 83 insertions(+), 27 deletions(-) create mode 100644 changelog/13055.feature.rst diff --git a/AUTHORS b/AUTHORS index 9629e00bcf..887c1a7526 100644 --- a/AUTHORS +++ b/AUTHORS @@ -56,6 +56,7 @@ Aviral Verma Aviv Palivoda Babak Keyvani Barney Gale +Bastian Krause Ben Brown Ben Gartner Ben Leith diff --git a/changelog/13055.feature.rst b/changelog/13055.feature.rst new file mode 100644 index 0000000000..7b7fdb71dc --- /dev/null +++ b/changelog/13055.feature.rst @@ -0,0 +1 @@ +``@pytest.mark.parametrize()`` and ``pytest.Metafunc.parametrize()`` now support the ``id_names`` argument enabling auto-generated test IDs consisting of parameter name=value pairs. diff --git a/doc/en/example/parametrize.rst b/doc/en/example/parametrize.rst index fa43308d04..d43a841bc9 100644 --- a/doc/en/example/parametrize.rst +++ b/doc/en/example/parametrize.rst @@ -111,12 +111,18 @@ the argument name: assert diff == expected - @pytest.mark.parametrize("a,b,expected", testdata, ids=["forward", "backward"]) + @pytest.mark.parametrize("a,b,expected", testdata, id_names=True) def test_timedistance_v1(a, b, expected): diff = a - b assert diff == expected + @pytest.mark.parametrize("a,b,expected", testdata, ids=["forward", "backward"]) + def test_timedistance_v2(a, b, expected): + diff = a - b + assert diff == expected + + def idfn(val): if isinstance(val, (datetime,)): # note this wouldn't show any hours/minutes/seconds @@ -124,7 +130,7 @@ the argument name: @pytest.mark.parametrize("a,b,expected", testdata, ids=idfn) - def test_timedistance_v2(a, b, expected): + def test_timedistance_v3(a, b, expected): diff = a - b assert diff == expected @@ -140,16 +146,19 @@ the argument name: ), ], ) - def test_timedistance_v3(a, b, expected): + def test_timedistance_v4(a, b, expected): diff = a - b assert diff == expected In ``test_timedistance_v0``, we let pytest generate the test IDs. -In ``test_timedistance_v1``, we specified ``ids`` as a list of strings which were +In ``test_timedistance_v1``, we let pytest generate the test IDs using argument +name/value pairs. + +In ``test_timedistance_v2``, we specified ``ids`` as a list of strings which were used as the test IDs. These are succinct, but can be a pain to maintain. -In ``test_timedistance_v2``, we specified ``ids`` as a function that can generate a +In ``test_timedistance_v3``, we specified ``ids`` as a function that can generate a string representation to make part of the test ID. So our ``datetime`` values use the label generated by ``idfn``, but because we didn't generate a label for ``timedelta`` objects, they are still using the default pytest representation: @@ -160,22 +169,24 @@ objects, they are still using the default pytest representation: =========================== test session starts ============================ platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y rootdir: /home/sweet/project - collected 8 items + collected 10 items - - - - - - - - ======================== 8 tests collected in 0.12s ======================== - -In ``test_timedistance_v3``, we used ``pytest.param`` to specify the test IDs + + + + + + + + + + ======================== 10 tests collected in 0.12s ======================= + +In ``test_timedistance_v4``, we used ``pytest.param`` to specify the test IDs together with the actual data, instead of listing them separately. A quick port of "testscenarios" diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 85e3cb0ae7..7cbadfa77e 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -884,18 +884,19 @@ class IdMaker: # Used only for clearer error messages. func_name: str | None - def make_unique_parameterset_ids(self) -> list[str]: + def make_unique_parameterset_ids(self, id_names: bool = False) -> list[str]: """Make a unique identifier for each ParameterSet, that may be used to identify the parametrization in a node ID. - Format is -...-[counter], where prm_x_token is + Format is [=]-...-[=][counter], + where prm_x is (only for id_names=True) and prm_x_token is - user-provided id, if given - else an id derived from the value, applicable for certain types - else The counter suffix is appended only in case a string wouldn't be unique otherwise. """ - resolved_ids = list(self._resolve_ids()) + resolved_ids = list(self._resolve_ids(id_names=id_names)) # All IDs must be unique! if len(resolved_ids) != len(set(resolved_ids)): # Record the number of occurrences of each ID. @@ -919,7 +920,7 @@ def make_unique_parameterset_ids(self) -> list[str]: ), f"Internal error: {resolved_ids=}" return resolved_ids - def _resolve_ids(self) -> Iterable[str]: + def _resolve_ids(self, id_names: bool = False) -> Iterable[str]: """Resolve IDs for all ParameterSets (may contain duplicates).""" for idx, parameterset in enumerate(self.parametersets): if parameterset.id is not None: @@ -930,8 +931,9 @@ def _resolve_ids(self) -> Iterable[str]: yield self._idval_from_value_required(self.ids[idx], idx) else: # ID not provided - generate it. + idval_func = self._idval_named if id_names else self._idval yield "-".join( - self._idval(val, argname, idx) + idval_func(val, argname, idx) for val, argname in zip(parameterset.values, self.argnames) ) @@ -948,6 +950,11 @@ def _idval(self, val: object, argname: str, idx: int) -> str: return idval return self._idval_from_argname(argname, idx) + def _idval_named(self, val: object, argname: str, idx: int) -> str: + """Make an ID in argname=value format for a parameter in a + ParameterSet.""" + return "=".join((argname, self._idval(val, argname, idx))) + def _idval_from_function(self, val: object, argname: str, idx: int) -> str | None: """Try to make an ID for a parameter in a ParameterSet using the user-provided id callable, if given.""" @@ -1141,6 +1148,7 @@ def parametrize( indirect: bool | Sequence[str] = False, ids: Iterable[object | None] | Callable[[Any], object | None] | None = None, scope: _ScopeName | None = None, + id_names: bool = False, *, _param_mark: Mark | None = None, ) -> None: @@ -1205,6 +1213,11 @@ def parametrize( The scope is used for grouping tests by parameter instances. It will also override any fixture-function defined scope, allowing to set a dynamic scope using test context or configuration. + + :param id_names: + Whether the argument names should be part of the auto-generated + ids. Defaults to ``False``. Must not be ``True`` if ``ids`` is + given. """ argnames, parametersets = ParameterSet._for_parametrize( argnames, @@ -1228,6 +1241,9 @@ def parametrize( else: scope_ = _find_parametrized_scope(argnames, self._arg2fixturedefs, indirect) + if id_names and ids is not None: + fail("'id_names' must not be combined with 'ids'", pytrace=False) + self._validate_if_using_arg_names(argnames, indirect) # Use any already (possibly) generated ids with parametrize Marks. @@ -1237,7 +1253,11 @@ def parametrize( ids = generated_ids ids = self._resolve_parameter_set_ids( - argnames, ids, parametersets, nodeid=self.definition.nodeid + argnames, + ids, + parametersets, + nodeid=self.definition.nodeid, + id_names=id_names, ) # Store used (possibly generated) ids with parametrize Marks. @@ -1322,6 +1342,7 @@ def _resolve_parameter_set_ids( ids: Iterable[object | None] | Callable[[Any], object | None] | None, parametersets: Sequence[ParameterSet], nodeid: str, + id_names: bool, ) -> list[str]: """Resolve the actual ids for the given parameter sets. @@ -1356,7 +1377,7 @@ def _resolve_parameter_set_ids( nodeid=nodeid, func_name=self.function.__name__, ) - return id_maker.make_unique_parameterset_ids() + return id_maker.make_unique_parameterset_ids(id_names=id_names) def _validate_ids( self, diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 4e7e441768..65db36405a 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -199,18 +199,28 @@ def find_scope(argnames, indirect): ) assert find_scope(["mixed_fix"], indirect=True) == Scope.Class - def test_parametrize_and_id(self) -> None: + @pytest.mark.parametrize("id_names", (False, True)) + def test_parametrize_and_id(self, id_names: bool) -> None: def func(x, y): pass metafunc = self.Metafunc(func) metafunc.parametrize("x", [1, 2], ids=["basic", "advanced"]) - metafunc.parametrize("y", ["abc", "def"]) + metafunc.parametrize("y", ["abc", "def"], id_names=id_names) ids = [x.id for x in metafunc._calls] - assert ids == ["basic-abc", "basic-def", "advanced-abc", "advanced-def"] + if id_names: + assert ids == [ + "basic-y=abc", + "basic-y=def", + "advanced-y=abc", + "advanced-y=def", + ] + else: + assert ids == ["basic-abc", "basic-def", "advanced-abc", "advanced-def"] - def test_parametrize_and_id_unicode(self) -> None: + @pytest.mark.parametrize("id_names", (False, True)) + def test_parametrize_and_id_unicode(self, id_names: bool) -> None: """Allow unicode strings for "ids" parameter in Python 2 (##1905)""" def func(x): @@ -221,6 +231,18 @@ def func(x): ids = [x.id for x in metafunc._calls] assert ids == ["basic", "advanced"] + def test_parametrize_with_bad_ids_name_combination(self) -> None: + def func(x): + pass + + metafunc = self.Metafunc(func) + + with pytest.raises( + fail.Exception, + match="'id_names' must not be combined with 'ids'", + ): + metafunc.parametrize("x", [1, 2], ids=["basic", "advanced"], id_names=True) + def test_parametrize_with_wrong_number_of_ids(self) -> None: def func(x, y): pass