Add 'directory' as a new predefined smoke scope

README.md

@@ -7,19 +7,25 @@ versions](https://img.shields.io/pypi/pyversions/pytest-smoke.svg)](https://pypi
 [![test](https://github.com/yugokato/pytest-smoke/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/yugokato/pytest-smoke/actions/workflows/test.yml?query=branch%3Amain)
 [![Code style ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://docs.astral.sh/ruff/)
 
-A `pytest` plugin that provides a quick way to perform smoke testing against a large test suite by limiting the 
-number of tests executed from each test function (or specified scope) to a value of `N`.  
-You can specify `N` as either a fixed number or a percentage, allowing you to scale the test execution down to a smaller 
-subset.
+`pytest-smoke` is a `pytest` plugin designed to quickly perform smoke testing on large test suites. It helps you scale 
+down test execution by running a smaller subset of tests from each function (or specified scope).
 
 
 ## Installation
 
-```
+```bash
 pip install pytest-smoke
 ```
 
 
+## Quick Start
+
+For a quick smoke test with all default options, simply run:
+```bash
+pytest --smoke
+```
+This will run a small subset of tests from your test suite to quickly check basic functionality.
+
 ## Usage
 
 This plugin provides the following command options:
@@ -30,14 +36,15 @@ $ pytest -h
 
 Smoke testing:
   --smoke=[N]           Run only N tests from each test function or specified scope.
-                        If N is explicitly provided to the option, it can be a number (e.g. 5) or a percentage (e.g. 10%).
-                        Otherwise, the default value of 1 will be applied.
+                        N can be a number (e.g. 5) or a percentage (e.g. 10%).
+                        If not provided, the default value is 1.
   --smoke-scope=SCOPE   Specify the scope at which the value of N from the above options is applied.
                         The plugin provides the following predefined scopes, as well as custom user-defined scopes via a hook:
                         - function: Applies to each test function (default)
                         - class: Applies to each test class
                         - auto: Applies function scope for test functions, class scope for test methods
                         - file: Applies to each test file
+                        - directory: Applies to each test directory
                         - all: Applies to the entire test suite
   --smoke-select-mode=MODE
                         Specify the mode for selecting tests from each scope.
@@ -48,7 +55,7 @@ Smoke testing:
 ```
 
 > [!NOTE]
-> - The `--smoke` option is required to use any `pytest-smoke` plugin functionality
+> - The `--smoke` option is always required to use any `pytest-smoke` plugin functionality
 > - The `--smoke-scope` and `--smoke-select-mode` options also support any custom values, as long as they are handled in the hook. See the "Hooks" section below
 > - You can override the plugin's default values for `N`, `SCOPE`, and `MODE` using INI options. See the "INI Options" section below
 > - When using the [pytest-xdist](https://pypi.org/project/pytest-xdist/) plugin for parallel testing, you can configure the `pytest-smoke` plugin to replace the default scheduler with a custom distribution algorithm that distributes tests based on the smoke scope
@@ -74,7 +81,8 @@ def test_something3(p):
     pass
 ```
 
-You can run smoke tests with the `--smoke` option. Here are some basic examples:
+You can run smoke tests on subsets of different sizes with the `--smoke` option.  
+Here are some basic examples:
 
 - Run only the first test from each test function
 ```

src/pytest_smoke/plugin.py

@@ -64,8 +64,8 @@ def pytest_addoption(parser: Parser):
         nargs="?",
         default=False,
         help="Run only N tests from each test function or specified scope.\n"
-        "If N is explicitly provided to the option, it can be a number (e.g. 5) or a percentage (e.g. 10%%).\n"
-        "Otherwise, the default value of 1 will be applied.",
+        "N can be a number (e.g. 5) or a percentage (e.g. 10%%).\n"
+        "If not provided, the default value is 1.",
     )
     group.addoption(
         "--smoke-scope",
@@ -80,6 +80,7 @@ def pytest_addoption(parser: Parser):
             f"- {SmokeScope.AUTO}: Applies {SmokeScope.FUNCTION} scope for test functions, "
             f"{SmokeScope.CLASS} scope for test methods\n"
             f"- {SmokeScope.FILE}: Applies to each test file\n"
+            f"- {SmokeScope.DIRECTORY}: Applies to each test directory\n"
             f"- {SmokeScope.ALL}: Applies to the entire test suite"
         ),
     )

src/pytest_smoke/types.py

@@ -21,6 +21,7 @@ class SmokeScope(StrEnum):
     CLASS = auto()
     AUTO = auto()
     FILE = auto()
+    DIRECTORY = auto()
     ALL = auto()
 
 

src/pytest_smoke/utils.py

@@ -61,7 +61,11 @@ def generate_group_id(item: Item, scope: str) -> Optional[str]:
     if not cls and scope == SmokeScope.CLASS:
         return
 
-    group_id = str(item.path or item.location[0])
+    file_path = item.path
+    if scope == SmokeScope.DIRECTORY:
+        return str(file_path.parent)
+
+    group_id = str(file_path)
     if scope == SmokeScope.FILE:
         return group_id
 

tests/conftest.py

@@ -46,9 +46,10 @@ def test_file_specs() -> list[TestFileSpec]:
             TestClassSpec("Test3", [TestFuncSpec(num_params=3), TestFuncSpec()], num_params=2),
         ]
     )
-    # file4: A test file with a mix of everything above
+    # file4: A test file with a mix of everything above, inside a sub directory
     test_file_spec4 = TestFileSpec(
-        [*test_file_spec1.test_specs, *test_file_spec2.test_specs, *test_file_spec3.test_specs]
+        [*test_file_spec1.test_specs, *test_file_spec2.test_specs, *test_file_spec3.test_specs],
+        test_dir="tests_something1",
     )
     return [test_file_spec1, test_file_spec2, test_file_spec3, test_file_spec4]
 
@@ -58,5 +59,8 @@ def generate_test_files(pytester: Pytester, test_file_specs: list[TestFileSpec])
     """Generate test files with given test file specs"""
     test_files = {}
     for i, test_file_spec in enumerate(test_file_specs, start=1):
-        test_files[f"test_{i}"] = generate_test_code(test_file_spec)
+        test_filename = f"test_{i}"
+        if test_dir := test_file_spec.test_dir:
+            test_filename = test_dir + os.sep + test_filename
+        test_files[test_filename] = generate_test_code(test_file_spec)
     pytester.makepyfile(**test_files)

tests/helper.py

@@ -6,6 +6,7 @@
 from dataclasses import dataclass, field
 from functools import wraps
 from itertools import chain
+from typing import Optional
 
 import pytest
 
@@ -27,6 +28,7 @@ class TestFileSpec:
     __test__ = False
 
     test_specs: list[TestFuncSpec | TestClassSpec] = field(default_factory=list)
+    test_dir: Optional[str] = None
 
 
 @dataclass
@@ -158,50 +160,42 @@ def get_num_tests_to_be_selected(test_file_specs: list[TestFileSpec], n: str | N
     :param n: N value given for the smoke option
     :param scope: Smoke scope value
     """
-    n = n or str(DEFAULT_N)
-    is_scale = n.endswith("%")
-    test_class_specs = get_test_class_specs(test_file_specs)
-    if is_scale:
-        scale = float(n[:-1])
-        if scope == SmokeScope.ALL:
-            num_expected_tests = scale_down(get_num_tests(*test_file_specs), scale)
-        elif scope == SmokeScope.FILE:
-            num_expected_tests = sum([int(scale_down(get_num_tests(*x.test_specs), scale)) for x in test_file_specs])
-        elif scope == SmokeScope.AUTO:
-            num_expected_tests = sum(
-                [
-                    *(int(scale_down(get_num_tests(x), scale)) for x in test_class_specs),
-                    *(
-                        int(scale_down(get_num_tests(x), scale))
-                        for x in get_test_func_specs(test_file_specs, exclude_class=True)
-                    ),
-                ]
-            )
-        elif scope == SmokeScope.CLASS:
-            num_expected_tests = sum([int(scale_down(get_num_tests(x), scale)) for x in test_class_specs])
+
+    def get_num_expected_tests_per_scope(*test_specs):
+        n_ = n or str(DEFAULT_N)
+        if n_.endswith("%"):
+            scale = float(n_[:-1])
+            return int(scale_down(get_num_tests(*test_specs), scale))
         else:
-            num_expected_tests = sum(
-                [int(scale_down(get_num_tests(x), scale)) for x in get_test_func_specs(test_file_specs)]
-            )
+            return min([int(n_), get_num_tests(*test_specs)])
+
+    test_class_specs = get_test_class_specs(test_file_specs) if scope in [SmokeScope.AUTO, SmokeScope.CLASS] else None
+    if scope == SmokeScope.ALL:
+        num_expected_tests = get_num_expected_tests_per_scope(*test_file_specs)
+    elif scope == SmokeScope.DIRECTORY:
+        test_specs_per_dir = {}
+        for test_file_spec in test_file_specs:
+            test_specs_per_dir.setdefault(test_file_spec.test_dir, []).extend(test_file_spec.test_specs)
+        num_expected_tests = sum(
+            get_num_expected_tests_per_scope(*test_specs) for test_specs in test_specs_per_dir.values()
+        )
+    elif scope == SmokeScope.FILE:
+        num_expected_tests = sum(get_num_expected_tests_per_scope(*x.test_specs) for x in test_file_specs)
+    elif scope == SmokeScope.AUTO:
+        num_expected_tests = sum(
+            [
+                *(get_num_expected_tests_per_scope(x) for x in test_class_specs),
+                *(
+                    get_num_expected_tests_per_scope(x)
+                    for x in get_test_func_specs(test_file_specs, exclude_class=True)
+                ),
+            ]
+        )
+    elif scope == SmokeScope.CLASS:
+        num_expected_tests = sum(get_num_expected_tests_per_scope(x) for x in test_class_specs)
     else:
-        if scope == SmokeScope.ALL:
-            num_expected_tests = min([int(n), get_num_tests(*test_file_specs)])
-        elif scope == SmokeScope.FILE:
-            num_expected_tests = sum([min([int(n), get_num_tests(*x.test_specs)]) for x in test_file_specs])
-        elif scope == SmokeScope.AUTO:
-            num_expected_tests = sum(
-                [
-                    *(min([int(n), get_num_tests(x)]) for x in test_class_specs),
-                    *(
-                        min([int(n), get_num_tests(x)])
-                        for x in get_test_func_specs(test_file_specs, exclude_class=True)
-                    ),
-                ]
-            )
-        elif scope == SmokeScope.CLASS:
-            num_expected_tests = sum([min([int(n), get_num_tests(x)]) for x in test_class_specs])
-        else:
-            num_expected_tests = sum([min([int(n), get_num_tests(x)]) for x in get_test_func_specs(test_file_specs)])
+        # default = function
+        num_expected_tests = sum(get_num_expected_tests_per_scope(x) for x in get_test_func_specs(test_file_specs))
 
     return num_expected_tests