Merge pull request #215 from scrapinghub/pyupgrade

Migrate to ruff

.github/workflows/tests-ubuntu.yml

@@ -38,10 +38,9 @@ jobs:
       run: |
         tox
     - name: coverage
-      uses: codecov/codecov-action@v4
-      if: ${{ success() }}
+      uses: codecov/codecov-action@v5
       with:
-        fail_ci_if_error: false
+        token: ${{ secrets.CODECOV_TOKEN }}
 
   check:
     runs-on: ubuntu-latest

.github/workflows/tests-windows.yml

@@ -38,7 +38,6 @@ jobs:
       run: |
         tox
     - name: coverage
-      uses: codecov/codecov-action@v4
-      if: ${{ success() }}
+      uses: codecov/codecov-action@v5
       with:
-        fail_ci_if_error: false
+        token: ${{ secrets.CODECOV_TOKEN }}

.pre-commit-config.yaml

@@ -1,25 +1,8 @@
+exclude: ^docs/tutorial-project/
 repos:
-  - hooks:
-      - id: black
-        language_version: python3
-        exclude: ^docs/tutorial-project/
-    repo: https://github.com/ambv/black
-    rev: 24.10.0
-  - hooks:
-      - id: isort
-        language_version: python3
-        exclude: ^docs/tutorial-project/
-    repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-  - hooks:
-      - id: flake8
-        language_version: python3
-        exclude: ^docs/tutorial-project/
-        additional_dependencies:
-          - flake8-bugbear
-          - flake8-comprehensions
-          - flake8-debugger
-          - flake8-docstrings
-          - flake8-string-format
-    repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.9.5
+  hooks:
+    - id: ruff
+      args: [ --fix ]
+    - id: ruff-format

docs/conf.py

@@ -4,8 +4,8 @@
 # full list see the documentation:
 # http://www.sphinx-doc.org/en/master/config
 
-import os
 import sys
+from pathlib import Path
 
 # -- Path setup --------------------------------------------------------------
 
@@ -14,7 +14,7 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 
 
-sys.path.insert(0, os.path.abspath(".."))
+sys.path.insert(0, str(Path(__file__).parent.parent))
 
 
 # -- Project information -----------------------------------------------------

pyproject.toml

@@ -17,9 +17,10 @@ filename = "docs/conf.py"
 filename = "setup.py"
 parse = "version\\s*=\\s*['\"](?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
 
-[tool.isort]
-profile = "black"
-multi_line_output = 3
+[tool.coverage.report]
+exclude_also = [
+    "if TYPE_CHECKING:",
+]
 
 [tool.mypy]
 ignore_missing_imports = true
@@ -30,3 +31,138 @@ module = "tests.po_lib_to_return.*"
 # Ignore mypy errors since the Page Objects contain arbitrary reference values
 # used for assertions which have varying types. This upsets mypy.
 ignore_errors = true
+
+[tool.pytest.ini_options]
+asyncio_mode = "strict"
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.ruff.lint]
+extend-select = [
+    # flake8-bugbear
+    "B",
+    # flake8-comprehensions
+    "C4",
+    # pydocstyle
+    "D",
+    # flake8-future-annotations
+    "FA",
+    # flynt
+    "FLY",
+    # refurb
+    "FURB",
+    # isort
+    "I",
+    # flake8-implicit-str-concat
+    "ISC",
+    # flake8-logging
+    "LOG",
+    # Perflint
+    "PERF",
+    # pygrep-hooks
+    "PGH",
+    # flake8-pie
+    "PIE",
+    # pylint
+    "PL",
+    # flake8-pytest-style
+    "PT",
+    # flake8-use-pathlib
+    "PTH",
+    # flake8-pyi
+    "PYI",
+    # flake8-quotes
+    "Q",
+    # flake8-return
+    "RET",
+    # flake8-raise
+    "RSE",
+    # Ruff-specific rules
+    "RUF",
+    # flake8-bandit
+    "S",
+    # flake8-simplify
+    "SIM",
+    # flake8-slots
+    "SLOT",
+    # flake8-debugger
+    "T10",
+    # flake8-type-checking
+    "TC",
+    # pyupgrade
+    "UP",
+    # pycodestyle warnings
+    "W",
+    # flake8-2020
+    "YTT",
+]
+ignore = [
+    # Missing docstring in public module
+    "D100",
+    # Missing docstring in public class
+    "D101",
+    # Missing docstring in public function
+    "D103",
+    # Missing docstring in public package
+    "D104",
+    # Missing docstring in magic method
+    "D105",
+    # Missing docstring in __init__
+    "D107",
+    # One-line docstring should fit on one line with quotes
+    "D200",
+    # No blank lines allowed after function docstring
+    "D202",
+    # 1 blank line required between summary line and description
+    "D205",
+    # Multi-line docstring closing quotes should be on a separate line
+    "D209",
+    # First line should end with a period
+    "D400",
+    # First line should be in imperative mood; try rephrasing
+    "D401",
+    # First line should not be the function's "signature"
+    "D402",
+    # Too many return statements
+    "PLR0911",
+    # Too many branches
+    "PLR0912",
+    # Too many arguments in function definition
+    "PLR0913",
+    # Too many statements
+    "PLR0915",
+    # Magic value used in comparison
+    "PLR2004",
+    # Mutable class attributes should be annotated with `typing.ClassVar`
+    "RUF012",
+    # Use of `assert` detected
+    "S101",
+    # Move application import {} into a type-checking block
+    # (The fix can break singledispatch() and/or andi.)
+    "TC001",
+    # Add `from __future__ import annotations` to simplify
+    # (The fix can break singledispatch() and/or andi.)
+    "FA100",
+]
+
+[tool.ruff.lint.per-file-ignores]
+# F401: Ignore "imported but unused" errors in __init__ files, as those
+# imports are there to expose submodule functions so they can be imported
+# directly from that module
+# F403: Ignore * imports in these files
+# D102: Missing docstring in public method
+# S: flake8-bandit
+# B018: Found useless expression.
+"web_poet/__init__.py" = ["F401", "F403"]
+"web_poet/exceptions/__init__.py" = ["F401", "F403"]
+"web_poet/page_inputs/__init__.py" = ["F401", "F403"]
+"web_poet/serialization/__init__.py" = ["F401", "F403"]
+"web_poet/testing/__init__.py" = ["F401", "F403"]
+"web_poet/testing/pytest.py" = ["D102"]
+"tests/**" = ["B018", "D102", "S"]
+"web_poet/example.py" = ["S"]
+# Documentation examples may miss docstrings and include undefined, example
+# names.
+"docs/**" = ["D102", "F821"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "pep257"

setup.py

@@ -1,14 +1,12 @@
-from setuptools import find_packages, setup
-
-with open("README.rst") as f:
-    long_description = f.read()
+from pathlib import Path
 
+from setuptools import find_packages, setup
 
 setup(
     name="web-poet",
     version="0.18.0",
     description="Zyte's Page Object pattern for web scraping",
-    long_description=long_description,
+    long_description=Path("README.rst").read_text(),
     long_description_content_type="text/x-rst",
     author="Zyte Group Ltd",
     author_email="[email protected]",

tests/conftest.py

@@ -1,4 +1,6 @@
-import os
+from __future__ import annotations
+
+from pathlib import Path
 
 import pytest
 
@@ -7,18 +9,16 @@
 pytest_plugins = ["pytester"]
 
 
-def read_fixture(path):
-    path = os.path.join(os.path.dirname(__file__), path)
-    with open(path, encoding="utf-8") as f:
-        return f.read()
+def read_fixture(path: str) -> str:
+    return (Path(__file__).parent / path).read_text(encoding="utf-8")
 
 
 @pytest.fixture
 def book_list_html():
     return read_fixture("fixtures/book_list.html")
 
 
-@pytest.fixture()
+@pytest.fixture
 def some_json_response():
     body = """
     {

tests/po_lib/__init__.py

@@ -2,7 +2,9 @@
 This package is just for overrides testing purposes.
 """
 
-from typing import Any, Dict, List, Type, Union
+from __future__ import annotations
+
+from typing import Any
 
 from url_matcher import Patterns
 
@@ -13,10 +15,10 @@
 
 
 class POBase(ItemPage):
-    expected_instead_of: Union[Type[ItemPage], List[Type[ItemPage]]]
+    expected_instead_of: type[ItemPage] | list[type[ItemPage]]
     expected_patterns: Patterns
     expected_to_return: Any = None
-    expected_meta: Dict[str, Any]
+    expected_meta: dict[str, Any]
 
 
 class POTopLevelOverriden1(ItemPage): ...
@@ -36,12 +38,12 @@ class POTopLevel1(POBase):
         Patterns(["example.com"]),
     ]
     expected_to_return = [None, None]
-    expected_meta = [{}, {}]  # type: ignore
+    expected_meta = [{}, {}]  # type: ignore[assignment]
 
 
 @handle_urls("example.com", instead_of=POTopLevelOverriden2)
 class POTopLevel2(POBase):
     expected_instead_of = POTopLevelOverriden2
     expected_patterns = Patterns(["example.com"])
     expected_to_return = None
-    expected_meta = {}  # type: ignore
+    expected_meta = {}

tests/po_lib/a_module.py

@@ -12,4 +12,4 @@ class POModule(POBase):
     expected_instead_of = POModuleOverriden
     expected_patterns = Patterns(["example.com"])
     expected_to_return = None
-    expected_meta = {"extra_arg": "foo"}  # type: ignore
+    expected_meta = {"extra_arg": "foo"}

tests/po_lib/nested_package/__init__.py

@@ -16,4 +16,4 @@ class PONestedPkg(POBase):
     expected_instead_of = PONestedPkgOverriden
     expected_patterns = Patterns(["example.com", "example.org"], ["/*.jpg|"])
     expected_to_return = None
-    expected_meta = {}  # type: ignore
+    expected_meta = {}