From 7950d784fde72fb73a9388ba2bd59b7bd8982b8f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Feb 2025 21:53:13 +0500 Subject: [PATCH 1/4] Add pyupgrade. --- .pre-commit-config.yaml | 6 ++ tests/po_lib/__init__.py | 8 ++- tests/po_lib_sub/__init__.py | 8 ++- tests/test_fields.py | 16 +++-- tests/test_input_validation.py | 4 +- tests/test_page_inputs.py | 14 ++-- tests/test_pages.py | 8 ++- tests/test_requests.py | 6 +- tests/test_serialization.py | 5 +- tests/test_testing.py | 14 ++-- tests/test_utils.py | 2 + .../po_lib_sub_not_imported/__init__.py | 8 ++- web_poet/_base.py | 10 +-- web_poet/annotated.py | 6 +- web_poet/example.py | 14 ++-- web_poet/exceptions/core.py | 6 +- web_poet/exceptions/http.py | 16 ++--- web_poet/fields.py | 26 +++---- web_poet/mixins.py | 4 +- web_poet/page_inputs/browser.py | 4 +- web_poet/page_inputs/client.py | 51 +++++++------- web_poet/page_inputs/http.py | 28 ++++---- web_poet/page_inputs/page_params.py | 3 + web_poet/page_inputs/response.py | 6 +- web_poet/page_inputs/stats.py | 6 +- web_poet/page_inputs/url.py | 4 +- web_poet/pages.py | 8 ++- web_poet/requests.py | 2 + web_poet/rules.py | 69 ++++++++----------- web_poet/serialization/api.py | 23 ++++--- web_poet/serialization/functions.py | 30 ++++---- web_poet/serialization/utils.py | 8 ++- web_poet/testing/__main__.py | 2 + web_poet/testing/fixture.py | 23 ++++--- web_poet/testing/itemadapter.py | 6 +- web_poet/testing/pytest.py | 30 ++++---- web_poet/testing/utils.py | 4 +- web_poet/utils.py | 14 ++-- 38 files changed, 270 insertions(+), 232 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 916ad476..162fbfbc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,3 +23,9 @@ repos: - flake8-string-format repo: https://github.com/pycqa/flake8 rev: 7.1.1 + - hooks: + - id: pyupgrade + args: [--py39-plus] + exclude: ^docs/tutorial-project/ + repo: https://github.com/asottile/pyupgrade + rev: v3.19.1 diff --git a/tests/po_lib/__init__.py b/tests/po_lib/__init__.py index 768ecf45..a24fc580 100644 --- a/tests/po_lib/__init__.py +++ b/tests/po_lib/__init__.py @@ -2,7 +2,9 @@ This package is just for overrides testing purposes. """ -from typing import Any, Dict, List, Type, Union +from __future__ import annotations + +from typing import Any from url_matcher import Patterns @@ -13,10 +15,10 @@ class POBase(ItemPage): - expected_instead_of: Union[Type[ItemPage], List[Type[ItemPage]]] + expected_instead_of: type[ItemPage] | list[type[ItemPage]] expected_patterns: Patterns expected_to_return: Any = None - expected_meta: Dict[str, Any] + expected_meta: dict[str, Any] class POTopLevelOverriden1(ItemPage): ... diff --git a/tests/po_lib_sub/__init__.py b/tests/po_lib_sub/__init__.py index 96fcf6d2..383f8304 100644 --- a/tests/po_lib_sub/__init__.py +++ b/tests/po_lib_sub/__init__.py @@ -2,7 +2,9 @@ external depedencies. """ -from typing import Any, Dict, Type +from __future__ import annotations + +from typing import Any from url_matcher import Patterns @@ -10,9 +12,9 @@ class POBase(ItemPage): - expected_instead_of: Type[ItemPage] + expected_instead_of: type[ItemPage] expected_patterns: Patterns - expected_meta: Dict[str, Any] + expected_meta: dict[str, Any] class POLibSubOverriden(ItemPage): ... diff --git a/tests/test_fields.py b/tests/test_fields.py index 1a6ccff6..1272ffe2 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import asyncio import random -from typing import Callable, List +from typing import Callable import attrs import pytest @@ -629,11 +631,11 @@ def proc1(s): class BasePage(ItemPage): class Processors: - f1: List[Callable] = [str.strip] + f1: list[Callable] = [str.strip] f2 = [str.strip] f3 = [str.strip] - f4: List[Callable] = [str.strip] - f5: List[Callable] = [str.strip] + f4: list[Callable] = [str.strip] + f5: list[Callable] = [str.strip] @field def f1(self): @@ -695,7 +697,7 @@ def desc(self): class Page(BasePage): class Processors(BasePage.Processors): - name: List[Callable] = [] + name: list[Callable] = [] @field def name(self): @@ -704,8 +706,8 @@ def name(self): class Page2(Page): class Processors(Page.Processors): - name: List[Callable] = [] - desc: List[Callable] = [] + name: list[Callable] = [] + desc: list[Callable] = [] @field def desc(self): diff --git a/tests/test_input_validation.py b/tests/test_input_validation.py index c599b98e..d730a1cf 100644 --- a/tests/test_input_validation.py +++ b/tests/test_input_validation.py @@ -1,6 +1,6 @@ """Test page object input validation scenarios.""" -from typing import Optional +from __future__ import annotations import attrs import pytest @@ -278,7 +278,7 @@ async def a(self): async def test_invalid_input_cross_api_caching(): @attrs.define class _Item(Item): - b: Optional[str] = None + b: str | None = None class Page(BaseCachingPage, Returns[_Item]): @field diff --git a/tests/test_page_inputs.py b/tests/test_page_inputs.py index 49e81847..1635ad78 100644 --- a/tests/test_page_inputs.py +++ b/tests/test_page_inputs.py @@ -64,7 +64,7 @@ def test_http_response_body_json() -> None: http_body = HttpResponseBody(b'{"foo": 123}') assert http_body.json() == {"foo": 123} - http_body = HttpResponseBody('{"ключ": "значение"}'.encode("utf8")) + http_body = HttpResponseBody('{"ключ": "значение"}'.encode()) assert http_body.json() == {"ключ": "значение"} @@ -295,7 +295,7 @@ def test_http_response_json() -> None: response = HttpResponse(url, body=b'{"key": "value"}') assert response.json() == {"key": "value"} - response = HttpResponse(url, body='{"ключ": "значение"}'.encode("utf8")) + response = HttpResponse(url, body='{"ключ": "значение"}'.encode()) assert response.json() == {"ключ": "значение"} @@ -343,19 +343,17 @@ def test_http_response_utf16() -> None: def test_explicit_encoding() -> None: - response = HttpResponse( - "http://www.example.com", "£".encode("utf-8"), encoding="utf-8" - ) + response = HttpResponse("http://www.example.com", "£".encode(), encoding="utf-8") assert response.encoding == "utf-8" assert response.text == "£" def test_explicit_encoding_invalid() -> None: response = HttpResponse( - "http://www.example.com", body="£".encode("utf-8"), encoding="latin1" + "http://www.example.com", body="£".encode(), encoding="latin1" ) assert response.encoding == "latin1" - assert response.text == "£".encode("utf-8").decode("latin1") + assert response.text == "£".encode().decode("latin1") def test_utf8_body_detection() -> None: @@ -386,7 +384,7 @@ def test_gb2312() -> None: def test_bom_encoding() -> None: response = HttpResponse( "http://www.example.com", - body=codecs.BOM_UTF8 + "🎉".encode("utf-8"), + body=codecs.BOM_UTF8 + "🎉".encode(), headers={"Content-type": "text/html; charset=cp1251"}, ) assert response.encoding == "utf-8" diff --git a/tests/test_pages.py b/tests/test_pages.py index 6aeb1051..dcfa23ba 100644 --- a/tests/test_pages.py +++ b/tests/test_pages.py @@ -1,4 +1,6 @@ -from typing import Generic, List, Optional, TypeVar +from __future__ import annotations + +from typing import Generic, TypeVar import attrs import pytest @@ -143,7 +145,7 @@ async def test_item_page_required_field_missing() -> None: @attrs.define class MyItem: name: str - price: Optional[float] + price: float | None class MyPage(ItemPage[MyItem]): @field @@ -267,7 +269,7 @@ class BookItem: @attrs.define class ListItem: - books: List[BookItem] + books: list[BookItem] @attrs.define class MyPage(ItemPage[ListItem]): diff --git a/tests/test_requests.py b/tests/test_requests.py index 1f294e06..b548aaf9 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1,4 +1,6 @@ -from typing import Callable, Union +from __future__ import annotations + +from typing import Callable from unittest import mock import pytest @@ -98,7 +100,7 @@ async def test_http_client_allow_status( method = getattr(client, method_name) - url_or_request: Union[str, HttpRequest] = "url" + url_or_request: str | HttpRequest = "url" if method_name == "execute": # NOTE: We're ignoring the type below due to the following mypy bugs: # - https://github.com/python/mypy/issues/10187 diff --git a/tests/test_serialization.py b/tests/test_serialization.py index d7844e9d..9e32d93f 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -1,4 +1,5 @@ -from typing import Annotated, Type +# from __future__ import annotations breaks some tests here +from typing import Annotated import attrs import pytest @@ -153,7 +154,7 @@ def __init__(self, value: int): def _serialize(o: C) -> SerializedLeafData: return {"bin": o.value.to_bytes((o.value.bit_length() + 7) // 8, "little")} - def _deserialize(t: Type[C], data: SerializedLeafData) -> C: + def _deserialize(t: type[C], data: SerializedLeafData) -> C: return t(int.from_bytes(data["bin"], "little")) register_serialization(_serialize, _deserialize) diff --git a/tests/test_testing.py b/tests/test_testing.py index 6d677ba3..9d94dfb8 100644 --- a/tests/test_testing.py +++ b/tests/test_testing.py @@ -1,8 +1,10 @@ +from __future__ import annotations + import datetime import json from collections import deque from pathlib import Path -from typing import Annotated, Any, Dict, Optional +from typing import Annotated, Any from zoneinfo import ZoneInfo import attrs @@ -31,7 +33,7 @@ def test_save_fixture(book_list_html_response, tmp_path) -> None: meta = {"foo": "bar", "frozen_time": "2022-01-01"} def _assert_fixture_files( - directory: Path, expected_meta: Optional[dict] = None + directory: Path, expected_meta: dict | None = None ) -> None: input_dir = directory / INPUT_DIR_NAME assert (input_dir / "HttpResponse-body.html").exists() @@ -218,13 +220,13 @@ def test_pytest_plugin_compare_item_fail(pytester, book_list_html_response) -> N @attrs.define(kw_only=True) class MetadataLocalTime(Metadata): - dateDownloadedLocal: Optional[str] = None + dateDownloadedLocal: str | None = None @attrs.define(kw_only=True) class ProductLocalTime(Product): # in newer zyte-common-items this should inherit from ProductMetadata - metadata: Optional[MetadataLocalTime] # type: ignore[assignment] + metadata: MetadataLocalTime | None # type: ignore[assignment] def _get_product_item(date: datetime.datetime) -> ProductLocalTime: @@ -250,10 +252,10 @@ async def to_item(self) -> Item: def _assert_frozen_item( frozen_time: datetime.datetime, - pytester: "pytest.Pytester", + pytester: pytest.Pytester, response: HttpResponse, *, - outcomes: Optional[Dict[str, int]] = None, + outcomes: dict[str, int] | None = None, ) -> None: # this makes an item with datetime fields corresponding to frozen_time item = ItemAdapter(_get_product_item(frozen_time)).asdict() diff --git a/tests/test_utils.py b/tests/test_utils.py index 9094fdbf..ed91e57c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import inspect import random diff --git a/tests_extra/po_lib_sub_not_imported/__init__.py b/tests_extra/po_lib_sub_not_imported/__init__.py index 9084af6d..70d42077 100644 --- a/tests_extra/po_lib_sub_not_imported/__init__.py +++ b/tests_extra/po_lib_sub_not_imported/__init__.py @@ -5,7 +5,9 @@ captures the rules annotated in this module if it was not imported. """ -from typing import Any, Dict, Type +from __future__ import annotations + +from typing import Any from url_matcher import Patterns @@ -13,9 +15,9 @@ class POBase: - expected_instead_of: Type[ItemPage] + expected_instead_of: type[ItemPage] expected_patterns: Patterns - expected_meta: Dict[str, Any] + expected_meta: dict[str, Any] class POLibSubOverridenNotImported: ... diff --git a/web_poet/_base.py b/web_poet/_base.py index cf669f2e..dffdb4dd 100644 --- a/web_poet/_base.py +++ b/web_poet/_base.py @@ -3,11 +3,13 @@ In general, users shouldn't import and use the contents of this module. """ -from typing import AnyStr, Dict, List, Tuple, Type, TypeVar, Union +from __future__ import annotations + +from typing import AnyStr, TypeVar, Union from multidict import CIMultiDict -_AnyStrDict = Dict[AnyStr, Union[AnyStr, List[AnyStr], Tuple[AnyStr, ...]]] +_AnyStrDict = dict[AnyStr, Union[AnyStr, list[AnyStr], tuple[AnyStr, ...]]] T_headers = TypeVar("T_headers", bound="_HttpHeaders") @@ -19,7 +21,7 @@ class _HttpHeaders(CIMultiDict): """ @classmethod - def from_name_value_pairs(cls: Type[T_headers], arg: List[Dict]) -> T_headers: + def from_name_value_pairs(cls: type[T_headers], arg: list[dict]) -> T_headers: """An alternative constructor for instantiation using a ``List[Dict]`` where the 'key' is the header name while the 'value' is the header value. @@ -35,7 +37,7 @@ def from_name_value_pairs(cls: Type[T_headers], arg: List[Dict]) -> T_headers: @classmethod def from_bytes_dict( - cls: Type[T_headers], arg: _AnyStrDict, encoding: str = "utf-8" + cls: type[T_headers], arg: _AnyStrDict, encoding: str = "utf-8" ) -> T_headers: """An alternative constructor for instantiation where the header-value pairs could be in raw bytes form. diff --git a/web_poet/annotated.py b/web_poet/annotated.py index 25d257ed..5ce28cc1 100644 --- a/web_poet/annotated.py +++ b/web_poet/annotated.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from dataclasses import dataclass -from typing import Any, Tuple +from typing import Any @dataclass @@ -17,7 +19,7 @@ class AnnotatedInstance: """ result: Any - metadata: Tuple[Any, ...] + metadata: tuple[Any, ...] def get_annotated_cls(self): """Returns a re-created :class:`typing.Annotated` type.""" diff --git a/web_poet/example.py b/web_poet/example.py index 384e57bf..0f3780ae 100644 --- a/web_poet/example.py +++ b/web_poet/example.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from asyncio import run -from typing import Any, Dict, Optional, Type +from typing import Any from warnings import warn import andi @@ -37,9 +39,9 @@ def _get_http_response(url: str) -> HttpResponse: def _get_page( url: str, - page_cls: Type[ItemPage], + page_cls: type[ItemPage], *, - page_params: Optional[Dict[Any, Any]] = None, + page_params: dict[Any, Any] | None = None, ) -> ItemPage: plan = andi.plan( page_cls, @@ -50,7 +52,7 @@ def _get_page( PageParams, }, ) - instances: Dict[Any, Any] = {} + instances: dict[Any, Any] = {} for fn_or_cls, kwargs_spec in plan: if fn_or_cls is HttpResponse: instances[fn_or_cls] = _get_http_response(url) @@ -65,9 +67,9 @@ def _get_page( def get_item( url: str, - item_cls: Type, + item_cls: type, *, - page_params: Optional[Dict[Any, Any]] = None, + page_params: dict[Any, Any] | None = None, ) -> Any: """Returns an item built from the specified URL using a page object class from the default registry. diff --git a/web_poet/exceptions/core.py b/web_poet/exceptions/core.py index 05e4ac2c..9553b4e3 100644 --- a/web_poet/exceptions/core.py +++ b/web_poet/exceptions/core.py @@ -7,7 +7,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from web_poet import HttpRequest @@ -57,9 +57,7 @@ class NoSavedHttpResponse(AssertionError): :type request: HttpRequest """ - def __init__( - self, msg: Optional[str] = None, request: Optional[HttpRequest] = None - ): + def __init__(self, msg: str | None = None, request: HttpRequest | None = None): self.request = request if msg is None: msg = f"There is no saved response available for this HTTP Request: {self.request}" diff --git a/web_poet/exceptions/http.py b/web_poet/exceptions/http.py index 95ac0520..3c2f0c22 100644 --- a/web_poet/exceptions/http.py +++ b/web_poet/exceptions/http.py @@ -6,7 +6,7 @@ operations. """ -from typing import Optional +from __future__ import annotations from web_poet.page_inputs.http import HttpRequest, HttpResponse @@ -24,11 +24,9 @@ class HttpError(IOError): :type request: HttpRequest """ - def __init__( - self, msg: Optional[str] = None, request: Optional[HttpRequest] = None - ): + def __init__(self, msg: str | None = None, request: HttpRequest | None = None): #: Request that triggered the exception. - self.request: Optional[HttpRequest] = request + self.request: HttpRequest | None = request if msg is None: msg = f"An Error ocurred when executing this HTTP Request: {self.request}" super().__init__(msg) @@ -68,12 +66,12 @@ class HttpResponseError(HttpError): def __init__( self, - msg: Optional[str] = None, - response: Optional[HttpResponse] = None, - request: Optional[HttpRequest] = None, + msg: str | None = None, + response: HttpResponse | None = None, + request: HttpRequest | None = None, ): #: Response that triggered the exception. - self.response: Optional[HttpResponse] = response + self.response: HttpResponse | None = response if msg is None: msg = f"Unexpected HTTP Response received: {self.response}" super().__init__(msg, request=request) diff --git a/web_poet/fields.py b/web_poet/fields.py index 85f12bf2..e893c852 100644 --- a/web_poet/fields.py +++ b/web_poet/fields.py @@ -3,10 +3,12 @@ into separate Page Object methods / properties. """ +from __future__ import annotations + import inspect from contextlib import suppress from functools import update_wrapper, wraps -from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar, cast +from typing import Callable, TypeVar, cast import attrs from itemadapter import ItemAdapter @@ -26,10 +28,10 @@ class FieldInfo: name: str #: field metadata - meta: Optional[dict] = None + meta: dict | None = None #: field processors - out: Optional[List[Callable]] = None + out: list[Callable] | None = None class FieldsMixin: @@ -58,8 +60,8 @@ def field( method=None, *, cached: bool = False, - meta: Optional[dict] = None, - out: Optional[List[Callable]] = None, + meta: dict | None = None, + out: list[Callable] | None = None, ): """ Page Object method decorated with ``@field`` decorator becomes a property, @@ -84,7 +86,7 @@ def __init__(self, method): f"@field decorator must be used on methods, {method!r} is decorated instead" ) self.original_method = method - self.name: Optional[str] = None + self.name: str | None = None def __set_name__(self, owner, name): self.name = name @@ -109,7 +111,7 @@ def __get__(self, instance, owner=None): processor_functions = getattr(owner.Processors, self.name, []) else: processor_functions = [] - processors: List[Tuple[Callable, bool]] = [] + processors: list[tuple[Callable, bool]] = [] for processor_function in processor_functions: takes_page = callable_has_parameter(processor_function, "page") processors.append((processor_function, takes_page)) @@ -170,7 +172,7 @@ def processed(page): return _field -def get_fields_dict(cls_or_instance) -> Dict[str, FieldInfo]: +def get_fields_dict(cls_or_instance) -> dict[str, FieldInfo]: """Return a dictionary with information about the fields defined for the class: keys are field names, and values are :class:`web_poet.fields.FieldInfo` instances. @@ -185,7 +187,7 @@ def get_fields_dict(cls_or_instance) -> Dict[str, FieldInfo]: # inference works properly if a non-default item_cls is passed; for dict # it's not working (return type is Any) async def item_from_fields( - obj, item_cls: Type[T] = dict, *, skip_nonitem_fields: bool = False # type: ignore[assignment] + obj, item_cls: type[T] = dict, *, skip_nonitem_fields: bool = False # type: ignore[assignment] ) -> T: """Return an item of ``item_cls`` type, with its attributes populated from the ``obj`` methods decorated with :class:`field` decorator. @@ -207,7 +209,7 @@ async def item_from_fields( def item_from_fields_sync( - obj, item_cls: Type[T] = dict, *, skip_nonitem_fields: bool = False # type: ignore[assignment] + obj, item_cls: type[T] = dict, *, skip_nonitem_fields: bool = False # type: ignore[assignment] ) -> T: """Synchronous version of :func:`item_from_fields`.""" field_names = list(get_fields_dict(obj)) @@ -217,8 +219,8 @@ def item_from_fields_sync( def _without_unsupported_field_names( - item_cls: type, field_names: List[str] -) -> List[str]: + item_cls: type, field_names: list[str] +) -> list[str]: item_field_names = ItemAdapter.get_field_names_from_class(item_cls) if item_field_names is None: # item_cls doesn't define field names upfront return field_names[:] diff --git a/web_poet/mixins.py b/web_poet/mixins.py index e37484fc..111c3f49 100644 --- a/web_poet/mixins.py +++ b/web_poet/mixins.py @@ -1,7 +1,7 @@ from __future__ import annotations import abc -from typing import TYPE_CHECKING, Protocol, Union +from typing import TYPE_CHECKING, Protocol from urllib.parse import urljoin import parsel @@ -69,7 +69,7 @@ def _base_url(self) -> str: self._cached_base_url = get_base_url(text, str(self.url)) # type: ignore[attr-defined] return self._cached_base_url - def urljoin(self, url: Union[str, RequestUrl, ResponseUrl]) -> RequestUrl: + def urljoin(self, url: str | RequestUrl | ResponseUrl) -> RequestUrl: """Return *url* as an absolute URL. If *url* is relative, it is made absolute relative to the base URL of diff --git a/web_poet/page_inputs/browser.py b/web_poet/page_inputs/browser.py index f9b02546..819b4d63 100644 --- a/web_poet/page_inputs/browser.py +++ b/web_poet/page_inputs/browser.py @@ -1,4 +1,4 @@ -from typing import Optional +from __future__ import annotations import attrs @@ -34,7 +34,7 @@ class BrowserResponse(SelectableMixin, UrlShortcutsMixin): url: ResponseUrl = attrs.field(converter=ResponseUrl) html: BrowserHtml = attrs.field(converter=BrowserHtml) - status: Optional[int] = attrs.field(default=None, kw_only=True) + status: int | None = attrs.field(default=None, kw_only=True) def _selector_input(self) -> str: return self.html diff --git a/web_poet/page_inputs/client.py b/web_poet/page_inputs/client.py index fec68913..3ff9958e 100644 --- a/web_poet/page_inputs/client.py +++ b/web_poet/page_inputs/client.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import asyncio import logging +from collections.abc import Iterable from dataclasses import dataclass from http import HTTPStatus -from typing import Callable, Dict, Iterable, List, Optional, Union, cast +from typing import Callable, Union, cast from web_poet.exceptions import HttpError, HttpResponseError from web_poet.exceptions.core import NoSavedHttpResponse @@ -19,10 +22,10 @@ logger = logging.getLogger(__name__) -_StrMapping = Dict[str, str] +_StrMapping = dict[str, str] _Headers = Union[_StrMapping, HttpRequestHeaders] _Body = Union[bytes, HttpRequestBody] -_StatusList = Union[str, int, List[Union[str, int]]] +_StatusList = Union[str, int, list[Union[str, int]]] @dataclass @@ -30,8 +33,8 @@ class _SavedResponseData: """Class for storing a request and its result.""" request: HttpRequest - response: Optional[HttpResponse] - exception: Optional[HttpError] = None + response: HttpResponse | None + exception: HttpError | None = None def fingerprint(self) -> str: """Return the request fingeprint.""" @@ -57,16 +60,16 @@ class HttpClient: def __init__( self, - request_downloader: Optional[Callable] = None, + request_downloader: Callable | None = None, *, save_responses: bool = False, return_only_saved_responses: bool = False, - responses: Optional[Iterable[_SavedResponseData]] = None, + responses: Iterable[_SavedResponseData] | None = None, ): self._request_downloader = request_downloader or _perform_request self.save_responses = save_responses self.return_only_saved_responses = return_only_saved_responses - self._saved_responses: Dict[str, _SavedResponseData] = { + self._saved_responses: dict[str, _SavedResponseData] = { data.fingerprint(): data for data in responses or [] } @@ -75,7 +78,7 @@ def _handle_status( response: HttpResponse, request: HttpRequest, *, - allow_status: Optional[_StatusList] = None, + allow_status: _StatusList | None = None, ) -> None: allow_status_normalized = list(map(str, as_list(allow_status))) allow_all_status = any( @@ -96,12 +99,12 @@ def _handle_status( async def request( self, - url: Union[str, _Url], + url: str | _Url, *, method: str = "GET", - headers: Optional[_Headers] = None, - body: Optional[_Body] = None, - allow_status: Optional[_StatusList] = None, + headers: _Headers | None = None, + body: _Body | None = None, + allow_status: _StatusList | None = None, ) -> HttpResponse: """This is a shortcut for creating an :class:`~.HttpRequest` instance and executing that request. @@ -133,10 +136,10 @@ async def request( async def get( self, - url: Union[str, _Url], + url: str | _Url, *, - headers: Optional[_Headers] = None, - allow_status: Optional[_StatusList] = None, + headers: _Headers | None = None, + allow_status: _StatusList | None = None, ) -> HttpResponse: """Similar to :meth:`~.HttpClient.request` but peforming a ``GET`` request. @@ -150,11 +153,11 @@ async def get( async def post( self, - url: Union[str, _Url], + url: str | _Url, *, - headers: Optional[_Headers] = None, - body: Optional[_Body] = None, - allow_status: Optional[_StatusList] = None, + headers: _Headers | None = None, + body: _Body | None = None, + allow_status: _StatusList | None = None, ) -> HttpResponse: """Similar to :meth:`~.HttpClient.request` but performing a ``POST`` request. @@ -168,7 +171,7 @@ async def post( ) async def execute( - self, request: HttpRequest, *, allow_status: Optional[_StatusList] = None + self, request: HttpRequest, *, allow_status: _StatusList | None = None ) -> HttpResponse: """Execute the specified :class:`~.HttpRequest` instance using the request implementation configured in the :class:`~.HttpClient` @@ -227,8 +230,8 @@ async def batch_execute( self, *requests: HttpRequest, return_exceptions: bool = False, - allow_status: Optional[_StatusList] = None, - ) -> List[Union[HttpResponse, HttpResponseError]]: + allow_status: _StatusList | None = None, + ) -> list[HttpResponse | HttpResponseError]: """Similar to :meth:`~.HttpClient.execute` but accepts a collection of :class:`~.HttpRequest` instances that would be batch executed. @@ -260,7 +263,7 @@ async def batch_execute( responses = await asyncio.gather( *coroutines, return_exceptions=return_exceptions ) - return cast(List[Union[HttpResponse, HttpResponseError]], responses) + return cast(list[Union[HttpResponse, HttpResponseError]], responses) def get_saved_responses(self) -> Iterable[_SavedResponseData]: """Return saved requests and responses.""" diff --git a/web_poet/page_inputs/http.py b/web_poet/page_inputs/http.py index c73f2831..72a86454 100644 --- a/web_poet/page_inputs/http.py +++ b/web_poet/page_inputs/http.py @@ -2,7 +2,7 @@ import json from hashlib import sha1 -from typing import Any, Optional, TypeVar, Union +from typing import Any, TypeVar from urllib.parse import urljoin import attrs @@ -38,11 +38,11 @@ class HttpRequestBody(bytes): class HttpResponseBody(bytes): """A container for holding the raw HTTP response body in bytes format.""" - def bom_encoding(self) -> Optional[str]: + def bom_encoding(self) -> str | None: """Returns the encoding from the byte order mark if present.""" return read_bom(self)[0] - def declared_encoding(self) -> Optional[str]: + def declared_encoding(self) -> str | None: """Return the encoding specified in meta tags in the html body, or ``None`` if no suitable encoding was found""" return html_body_declared_encoding(self) @@ -113,7 +113,7 @@ class HttpResponseHeaders(_HttpHeaders): the API spec of :class:`multidict.CIMultiDict`. """ - def declared_encoding(self) -> Optional[str]: + def declared_encoding(self) -> str | None: """Return encoding detected from the Content-Type header, or None if encoding is not found""" content_type = self.get("Content-Type", "") @@ -139,7 +139,7 @@ class HttpRequest: factory=HttpRequestBody, converter=HttpRequestBody, kw_only=True ) - def urljoin(self, url: Union[str, _RequestUrl, _ResponseUrl]) -> _RequestUrl: + def urljoin(self, url: str | _RequestUrl | _ResponseUrl) -> _RequestUrl: """Return *url* as an absolute URL. If *url* is relative, it is made absolute relative to :attr:`url`.""" @@ -171,14 +171,14 @@ class HttpResponse(SelectableMixin, UrlShortcutsMixin): url: _ResponseUrl = attrs.field(converter=_ResponseUrl) body: HttpResponseBody = attrs.field(converter=HttpResponseBody) - status: Optional[int] = attrs.field(default=None, kw_only=True) + status: int | None = attrs.field(default=None, kw_only=True) headers: HttpResponseHeaders = attrs.field( factory=HttpResponseHeaders, converter=HttpResponseHeaders, kw_only=True ) - _encoding: Optional[str] = attrs.field(default=None, kw_only=True) + _encoding: str | None = attrs.field(default=None, kw_only=True) _DEFAULT_ENCODING = "ascii" - _cached_text: Optional[str] = None + _cached_text: str | None = None @property def text(self) -> str: @@ -201,7 +201,7 @@ def _selector_input(self) -> str: return self.text @property - def encoding(self) -> Optional[str]: + def encoding(self) -> str | None: """Encoding of the response""" return ( self._encoding @@ -217,19 +217,19 @@ def json(self) -> Any: return self.body.json() @memoizemethod_noargs - def _body_bom_encoding(self) -> Optional[str]: + def _body_bom_encoding(self) -> str | None: return self.body.bom_encoding() @memoizemethod_noargs - def _headers_declared_encoding(self) -> Optional[str]: + def _headers_declared_encoding(self) -> str | None: return self.headers.declared_encoding() @memoizemethod_noargs - def _body_declared_encoding(self) -> Optional[str]: + def _body_declared_encoding(self) -> str | None: return self.body.declared_encoding() @memoizemethod_noargs - def _body_inferred_encoding(self) -> Optional[str]: + def _body_inferred_encoding(self) -> str | None: content_type = self.headers.get("Content-Type", "") body_encoding, text = html_to_unicode( content_type, @@ -242,7 +242,7 @@ def _body_inferred_encoding(self) -> Optional[str]: self._cached_text = text return body_encoding - def _auto_detect_fun(self, body: bytes) -> Optional[str]: + def _auto_detect_fun(self, body: bytes) -> str | None: for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"): try: body.decode(enc) diff --git a/web_poet/page_inputs/page_params.py b/web_poet/page_inputs/page_params.py index b2240b27..555e8420 100644 --- a/web_poet/page_inputs/page_params.py +++ b/web_poet/page_inputs/page_params.py @@ -1,3 +1,6 @@ +from __future__ import annotations + + class PageParams(dict): """Container class that could contain any arbitrary data to be passed into a Page Object. diff --git a/web_poet/page_inputs/response.py b/web_poet/page_inputs/response.py index 2b7d6a4c..226fdf3a 100644 --- a/web_poet/page_inputs/response.py +++ b/web_poet/page_inputs/response.py @@ -1,4 +1,4 @@ -from typing import Optional, Union +from __future__ import annotations import attrs @@ -12,7 +12,7 @@ class AnyResponse(SelectableMixin, UrlShortcutsMixin): """A container that holds either :class:`~.BrowserResponse` or :class:`~.HttpResponse`.""" - response: Union[BrowserResponse, HttpResponse] + response: BrowserResponse | HttpResponse @property def url(self) -> ResponseUrl: @@ -27,7 +27,7 @@ def text(self) -> str: return self.response.text @property - def status(self) -> Optional[int]: + def status(self) -> int | None: """The int status code of the HTTP response, if available.""" return self.response.status diff --git a/web_poet/page_inputs/stats.py b/web_poet/page_inputs/stats.py index c8d35e55..a7d59892 100644 --- a/web_poet/page_inputs/stats.py +++ b/web_poet/page_inputs/stats.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from abc import ABC, abstractmethod -from typing import Any, Dict, Union +from typing import Any, Union StatNum = Union[int, float] @@ -24,7 +26,7 @@ class DummyStatCollector(StatCollector): storage is not necessary.""" def __init__(self): - self._stats: Dict[str, Any] = {} + self._stats: dict[str, Any] = {} def set(self, key: str, value: Any) -> None: # noqa: D102 self._stats[key] = value diff --git a/web_poet/page_inputs/url.py b/web_poet/page_inputs/url.py index a27382b4..a4376790 100644 --- a/web_poet/page_inputs/url.py +++ b/web_poet/page_inputs/url.py @@ -1,10 +1,10 @@ -from typing import Union +from __future__ import annotations class _Url: """Base URL class.""" - def __init__(self, url: Union[str, "_Url"]): + def __init__(self, url: str | _Url): if not isinstance(url, (str, _Url)): raise TypeError( f"`url` must be a str or an instance of _Url, " diff --git a/web_poet/pages.py b/web_poet/pages.py index 99a0f315..c8d56187 100644 --- a/web_poet/pages.py +++ b/web_poet/pages.py @@ -1,8 +1,10 @@ +from __future__ import annotations + import abc import inspect from contextlib import suppress from functools import wraps -from typing import Any, Generic, Optional, TypeVar, overload +from typing import Any, Generic, TypeVar, overload import attr import parsel @@ -59,10 +61,10 @@ def get_item_cls(cls: type, default: type) -> type: ... @overload -def get_item_cls(cls: type, default: None) -> Optional[type]: ... +def get_item_cls(cls: type, default: None) -> type | None: ... -def get_item_cls(cls: type, default: Optional[type] = None) -> Optional[type]: +def get_item_cls(cls: type, default: type | None = None) -> type | None: param = get_generic_param(cls, Returns) return param or default diff --git a/web_poet/requests.py b/web_poet/requests.py index 240aad6d..2a8fd1ac 100644 --- a/web_poet/requests.py +++ b/web_poet/requests.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from contextvars import ContextVar diff --git a/web_poet/rules.py b/web_poet/rules.py index 8f54b003..b086c861 100644 --- a/web_poet/rules.py +++ b/web_poet/rules.py @@ -1,24 +1,13 @@ -from __future__ import annotations # https://www.python.org/dev/peps/pep-0563/ +from __future__ import annotations import importlib import importlib.util import pkgutil import warnings from collections import defaultdict, deque +from collections.abc import Generator, Iterable, Mapping from operator import attrgetter -from typing import ( - Any, - DefaultDict, - Dict, - Generator, - Iterable, - List, - Mapping, - Optional, - Type, - TypeVar, - Union, -) +from typing import Any, DefaultDict, TypeVar, Union import attrs from url_matcher import Patterns, URLMatcher @@ -83,10 +72,10 @@ class ApplyRule: """ for_patterns: Patterns = attrs.field(converter=str_to_pattern) - use: Type[ItemPage] = attrs.field(kw_only=True) - instead_of: Optional[Type[ItemPage]] = attrs.field(default=None, kw_only=True) - to_return: Optional[Type[Any]] = attrs.field(default=None, kw_only=True) - meta: Dict[str, Any] = attrs.field(factory=dict, kw_only=True) + use: type[ItemPage] = attrs.field(kw_only=True) + instead_of: type[ItemPage] | None = attrs.field(default=None, kw_only=True) + to_return: type[Any] | None = attrs.field(default=None, kw_only=True) + meta: dict[str, Any] = attrs.field(factory=dict, kw_only=True) def __hash__(self): return hash((self.for_patterns, self.use, self.instead_of, self.to_return)) @@ -124,12 +113,12 @@ class ExampleComProductPage(WebPage[Product]): rules to separate it from the ``default_registry``. """ - def __init__(self, *, rules: Optional[Iterable[ApplyRule]] = None): - self._rules: Dict[int, ApplyRule] = {} - self._overrides_matchers: DefaultDict[Optional[Type[ItemPage]], URLMatcher] = ( + def __init__(self, *, rules: Iterable[ApplyRule] | None = None): + self._rules: dict[int, ApplyRule] = {} + self._overrides_matchers: DefaultDict[type[ItemPage] | None, URLMatcher] = ( defaultdict(URLMatcher) ) - self._item_matchers: DefaultDict[Optional[Type], URLMatcher] = defaultdict( + self._item_matchers: DefaultDict[type | None, URLMatcher] = defaultdict( URLMatcher ) @@ -196,7 +185,7 @@ def _format_list(cls, objects: Iterable) -> str: @classmethod def from_override_rules( - cls: Type[RulesRegistryTV], rules: List[ApplyRule] + cls: type[RulesRegistryTV], rules: list[ApplyRule] ) -> RulesRegistryTV: """Deprecated. Use ``RulesRegistry(rules=...)`` instead.""" msg = ( @@ -210,10 +199,10 @@ def handle_urls( self, include: Strings, *, - overrides: Optional[Type[ItemPage]] = None, - instead_of: Optional[Type[ItemPage]] = None, - to_return: Optional[Type] = None, - exclude: Optional[Strings] = None, + overrides: type[ItemPage] | None = None, + instead_of: type[ItemPage] | None = None, + to_return: type | None = None, + exclude: Strings | None = None, priority: int = 500, **kwargs, ): @@ -276,7 +265,7 @@ def wrapper(cls): return wrapper - def get_rules(self) -> List[ApplyRule]: + def get_rules(self) -> list[ApplyRule]: """Return all the :class:`~.ApplyRule` that were declared using the ``@handle_urls`` decorator. @@ -288,13 +277,13 @@ def get_rules(self) -> List[ApplyRule]: """ return list(self._rules.values()) - def get_overrides(self) -> List[ApplyRule]: + def get_overrides(self) -> list[ApplyRule]: """Deprecated, use :meth:`~.RulesRegistry.get_rules` instead.""" msg = "The 'get_overrides' method is deprecated. Use 'get_rules' instead." warnings.warn(msg, DeprecationWarning, stacklevel=2) return self.get_rules() - def search(self, **kwargs) -> List[ApplyRule]: + def search(self, **kwargs) -> list[ApplyRule]: """Return any :class:`ApplyRule` from the registry that matches with all the provided attributes. @@ -350,15 +339,15 @@ def finder(rule: ApplyRule): results.append(rule) return results - def search_overrides(self, **kwargs) -> List[ApplyRule]: + def search_overrides(self, **kwargs) -> list[ApplyRule]: """Deprecated, use :meth:`~.RulesRegistry.search` instead.""" msg = "The 'search_overrides' method is deprecated. Use 'search' instead." warnings.warn(msg, DeprecationWarning, stacklevel=2) return self.search(**kwargs) def _match_url_for_page_object( - self, url: Union[_Url, str], matcher: Optional[URLMatcher] = None - ) -> Optional[Type[ItemPage]]: + self, url: _Url | str, matcher: URLMatcher | None = None + ) -> type[ItemPage] | None: """Returns the page object to use based on the URL and URLMatcher.""" if not url or matcher is None: return None @@ -367,13 +356,11 @@ def _match_url_for_page_object( if rule_id is not None: return self._rules[rule_id].use - def overrides_for( - self, url: Union[_Url, str] - ) -> Mapping[Type[ItemPage], Type[ItemPage]]: + def overrides_for(self, url: _Url | str) -> Mapping[type[ItemPage], type[ItemPage]]: """Finds all of the page objects associated with the given URL and returns a Mapping where the 'key' represents the page object that is **overridden** by the page object in 'value'.""" - result: Dict[Type[ItemPage], Type[ItemPage]] = {} + result: dict[type[ItemPage], type[ItemPage]] = {} for replaced_page, matcher in self._overrides_matchers.items(): if replaced_page is None: continue @@ -382,9 +369,7 @@ def overrides_for( result[replaced_page] = page return result - def page_cls_for_item( - self, url: Union[_Url, str], item_cls: Type - ) -> Optional[Type]: + def page_cls_for_item(self, url: _Url | str, item_cls: type) -> type | None: """Return the page object class associated with the given URL that's able to produce the given ``item_cls``.""" if item_cls is None: @@ -393,8 +378,8 @@ def page_cls_for_item( return self._match_url_for_page_object(url, matcher) def top_rules_for_item( - self, url: Union[_Url, str], item_cls: Type - ) -> Generator[ApplyRule, None, None]: + self, url: _Url | str, item_cls: type + ) -> Generator[ApplyRule]: """Iterates the top rules that apply for *url* and *item_cls*. If multiple rules score the same, multiple rules are iterated. This may diff --git a/web_poet/serialization/api.py b/web_poet/serialization/api.py index cd36c54b..e74517ac 100644 --- a/web_poet/serialization/api.py +++ b/web_poet/serialization/api.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import os +from collections.abc import Iterable from functools import singledispatch from importlib import import_module from pathlib import Path -from typing import Any, Callable, Dict, Iterable, Tuple, Type, TypeVar, Union +from typing import Any, Callable, TypeVar import andi from andi.typeutils import strip_annotated @@ -14,22 +17,22 @@ from web_poet.utils import get_fq_class_name # represents a leaf dependency of any type serialized as a set of files -SerializedLeafData = Dict[str, bytes] +SerializedLeafData = dict[str, bytes] # represents a set of leaf dependencies of different types -SerializedData = Dict[str, SerializedLeafData] +SerializedData = dict[str, SerializedLeafData] T = TypeVar("T") InjectableT = TypeVar("InjectableT", bound=Injectable) SerializeFunction = Callable[[Any], SerializedLeafData] -DeserializeFunction = Callable[[Type[T], SerializedLeafData], T] +DeserializeFunction = Callable[[type[T], SerializedLeafData], T] class SerializedDataFileStorage: - def __init__(self, directory: Union[str, os.PathLike]) -> None: + def __init__(self, directory: str | os.PathLike) -> None: super().__init__() self.directory: Path = Path(directory) @staticmethod - def _split_file_name(file_name: str) -> Tuple[str, str]: + def _split_file_name(file_name: str) -> tuple[str, str]: """Extract the type name and the type-specific suffix from a file name. >>> SerializedDataFileStorage._split_file_name("TypeName.ext") @@ -94,7 +97,7 @@ def serialize_leaf(o: Any) -> SerializedLeafData: raise NotImplementedError(f"Serialization for {type(o)} is not implemented") -def _deserialize_leaf_base(cls: Type[Any], data: SerializedLeafData) -> Any: +def _deserialize_leaf_base(cls: type[Any], data: SerializedLeafData) -> Any: raise NotImplementedError(f"Deserialization for {cls} is not implemented") @@ -109,7 +112,7 @@ def register_serialization( f_serialize.f_deserialize = f_deserialize # type: ignore[attr-defined] -def deserialize_leaf(cls: Type[T], data: SerializedLeafData) -> T: +def deserialize_leaf(cls: type[T], data: SerializedLeafData) -> T: f_ser: SerializeFunction = serialize_leaf.dispatch(cls) # type: ignore[attr-defined] return f_ser.f_deserialize(cls, data) # type: ignore[attr-defined] @@ -189,8 +192,8 @@ def load_class(type_name: str) -> type: return result -def deserialize(cls: Type[InjectableT], data: SerializedData) -> InjectableT: - deps: Dict[Callable, Any] = {} +def deserialize(cls: type[InjectableT], data: SerializedData) -> InjectableT: + deps: dict[Callable, Any] = {} for dep_type_name, dep_data in data.items(): if dep_type_name.startswith("AnnotatedInstance "): diff --git a/web_poet/serialization/functions.py b/web_poet/serialization/functions.py index 9490084b..f72eb9c7 100644 --- a/web_poet/serialization/functions.py +++ b/web_poet/serialization/functions.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import json -from typing import Any, Dict, List, Optional, Type, cast +from typing import Any, cast from .. import ( HttpClient, @@ -40,7 +42,7 @@ def _serialize_HttpRequest(o: HttpRequest) -> SerializedLeafData: def _deserialize_HttpRequest( - cls: Type[HttpRequest], data: SerializedLeafData + cls: type[HttpRequest], data: SerializedLeafData ) -> HttpRequest: body = HttpRequestBody(data.get("body.txt", b"")) info = json.loads(data["info.json"]) @@ -69,7 +71,7 @@ def _serialize_HttpResponse(o: HttpResponse) -> SerializedLeafData: def _deserialize_HttpResponse( - cls: Type[HttpResponse], data: SerializedLeafData + cls: type[HttpResponse], data: SerializedLeafData ) -> HttpResponse: body = HttpResponseBody(data["body.html"]) info = json.loads(data["info.json"]) @@ -90,7 +92,7 @@ def _serialize_HttpResponseBody(o: HttpResponseBody) -> SerializedLeafData: def _deserialize_HttpResponseBody( - cls: Type[HttpResponseBody], data: SerializedLeafData + cls: type[HttpResponseBody], data: SerializedLeafData ) -> HttpResponseBody: return cls(data["html"]) @@ -102,7 +104,7 @@ def _serialize__Url(o: _Url) -> SerializedLeafData: return {"txt": str(o).encode()} -def _deserialize__Url(cls: Type[_Url], data: SerializedLeafData) -> _Url: +def _deserialize__Url(cls: type[_Url], data: SerializedLeafData) -> _Url: return cls(data["txt"].decode()) @@ -129,13 +131,13 @@ def _serialize_HttpClient(o: HttpClient) -> SerializedLeafData: def _deserialize_HttpClient( - cls: Type[HttpClient], data: SerializedLeafData + cls: type[HttpClient], data: SerializedLeafData ) -> HttpClient: - responses: List[_SavedResponseData] = [] + responses: list[_SavedResponseData] = [] - serialized_requests: Dict[str, SerializedLeafData] = {} - serialized_responses: Dict[str, SerializedLeafData] = {} - serialized_exceptions: Dict[str, SerializedLeafData] = {} + serialized_requests: dict[str, SerializedLeafData] = {} + serialized_responses: dict[str, SerializedLeafData] = {} + serialized_exceptions: dict[str, SerializedLeafData] = {} for k, v in data.items(): if k == "exists": continue @@ -160,7 +162,7 @@ def _deserialize_HttpClient( response = deserialize_leaf(HttpResponse, serialized_response) else: response = None - exception: Optional[HttpError] + exception: HttpError | None if serialized_exception: exc_data = json.loads(serialized_exception["json"]) exception = cast(HttpError, _exception_from_dict(exc_data)) @@ -179,7 +181,7 @@ def _serialize_PageParams(o: PageParams) -> SerializedLeafData: def _deserialize_PageParams( - cls: Type[PageParams], data: SerializedLeafData + cls: type[PageParams], data: SerializedLeafData ) -> PageParams: return cls(json.loads(data["json"])) @@ -191,7 +193,7 @@ def _serialize_Stats(o: Stats) -> SerializedLeafData: return {} -def _deserialize_Stats(cls: Type[Stats], data: SerializedLeafData) -> Stats: +def _deserialize_Stats(cls: type[Stats], data: SerializedLeafData) -> Stats: return cls() @@ -210,7 +212,7 @@ def _serialize_AnnotatedInstance(o: AnnotatedInstance) -> SerializedLeafData: def _deserialize_AnnotatedInstance( - cls: Type[AnnotatedInstance], data: SerializedLeafData + cls: type[AnnotatedInstance], data: SerializedLeafData ) -> AnnotatedInstance: metadata = json.loads(data["metadata.json"]) result_type = load_class(data["result_type.txt"].decode()) diff --git a/web_poet/serialization/utils.py b/web_poet/serialization/utils.py index d2b1d9b3..0b638950 100644 --- a/web_poet/serialization/utils.py +++ b/web_poet/serialization/utils.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import json -from typing import Any, Dict +from typing import Any from web_poet.serialization.api import _get_name_for_class, load_class -def _exception_to_dict(ex: Exception) -> Dict[str, Any]: +def _exception_to_dict(ex: Exception) -> dict[str, Any]: """Serialize an exception. Only the exception type and the first argument are saved. @@ -15,7 +17,7 @@ def _exception_to_dict(ex: Exception) -> Dict[str, Any]: } -def _exception_from_dict(data: Dict[str, Any]) -> Exception: +def _exception_from_dict(data: dict[str, Any]) -> Exception: """Deserialize an exception. Only the exception type and the first argument are restored. diff --git a/web_poet/testing/__main__.py b/web_poet/testing/__main__.py index f72e133e..6ab8df97 100644 --- a/web_poet/testing/__main__.py +++ b/web_poet/testing/__main__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import sys from pathlib import Path diff --git a/web_poet/testing/fixture.py b/web_poet/testing/fixture.py index ee26edef..83328703 100644 --- a/web_poet/testing/fixture.py +++ b/web_poet/testing/fixture.py @@ -1,10 +1,13 @@ +from __future__ import annotations + import asyncio import datetime import json import logging import os +from collections.abc import Iterable from pathlib import Path -from typing import Any, Iterable, Optional, Type, TypeVar, Union, cast +from typing import Any, TypeVar, cast from zoneinfo import ZoneInfo import dateutil.parser @@ -44,7 +47,7 @@ FixtureT = TypeVar("FixtureT", bound="Fixture") -def _get_available_filename(template: str, directory: Union[str, os.PathLike]) -> str: +def _get_available_filename(template: str, directory: str | os.PathLike) -> str: i = 1 while True: result = Path(directory, template.format(i)) @@ -58,7 +61,7 @@ class Fixture: def __init__(self, path: Path) -> None: self.path = path - self._output_error: Optional[Exception] = None + self._output_error: Exception | None = None @property def type_name(self) -> str: @@ -118,11 +121,11 @@ def get_meta(self) -> dict: meta_dict["adapter"] = load_class(meta_dict["adapter"]) return meta_dict - def _get_adapter_cls(self) -> Type[ItemAdapter]: + def _get_adapter_cls(self) -> type[ItemAdapter]: cls = self.get_meta().get("adapter") if not cls: return WebPoetTestItemAdapter - return cast(Type[ItemAdapter], cls) + return cast(type[ItemAdapter], cls) def _get_output(self) -> dict: page = self.get_page() @@ -137,7 +140,7 @@ def get_output(self) -> dict: """ try: meta = self.get_meta() - frozen_time: Optional[str] = meta.get("frozen_time") + frozen_time: str | None = meta.get("frozen_time") if frozen_time: frozen_time_parsed = self._parse_frozen_time(frozen_time) with time_machine.travel(frozen_time_parsed): @@ -244,13 +247,13 @@ def assert_toitem_exception(self) -> None: @classmethod def save( - cls: Type[FixtureT], - base_directory: Union[str, os.PathLike], + cls: type[FixtureT], + base_directory: str | os.PathLike, *, inputs: Iterable[Any], item: Any = None, - exception: Optional[Exception] = None, - meta: Optional[dict] = None, + exception: Exception | None = None, + meta: dict | None = None, fixture_name=None, ) -> FixtureT: """Save and return a fixture.""" diff --git a/web_poet/testing/itemadapter.py b/web_poet/testing/itemadapter.py index 193ed7ff..9fac7625 100644 --- a/web_poet/testing/itemadapter.py +++ b/web_poet/testing/itemadapter.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from collections import deque -from typing import Deque, Type +from typing import Deque from itemadapter import ItemAdapter from itemadapter.adapter import ( @@ -16,7 +18,7 @@ class WebPoetTestItemAdapter(ItemAdapter): """A default adapter implementation""" # In case the user changes ItemAdapter.ADAPTER_CLASSES it's copied here. - ADAPTER_CLASSES: Deque[Type[AdapterInterface]] = deque( + ADAPTER_CLASSES: Deque[type[AdapterInterface]] = deque( [ ScrapyItemAdapter, DictAdapter, diff --git a/web_poet/testing/pytest.py b/web_poet/testing/pytest.py index 4bc58490..71a96b01 100644 --- a/web_poet/testing/pytest.py +++ b/web_poet/testing/pytest.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import operator +from collections.abc import Iterable from pathlib import Path -from typing import Iterable, List, Optional, Set, Union import pytest @@ -31,12 +33,12 @@ class WebPoetFile(pytest.File, _PathCompatMixin): """Represents a directory containing test subdirectories for one Page Object.""" @staticmethod - def sorted(items: List["WebPoetCollector"]) -> List["WebPoetCollector"]: + def sorted(items: list[WebPoetCollector]) -> list[WebPoetCollector]: """Sort the test list by the test name.""" return sorted(items, key=operator.attrgetter("name")) - def collect(self) -> Iterable[Union[pytest.Item, pytest.Collector]]: # noqa: D102 - result: List[WebPoetCollector] = [] + def collect(self) -> Iterable[pytest.Item | pytest.Collector]: # noqa: D102 + result: list[WebPoetCollector] = [] path = self._path for entry in path.iterdir(): if entry.is_dir(): @@ -54,10 +56,10 @@ class WebPoetCollector(pytest.Collector, _PathCompatMixin): """Represents a directory containing one test.""" def __init__(self, name: str, parent=None, **kwargs) -> None: - super(WebPoetCollector, self).__init__(name, parent, **kwargs) + super().__init__(name, parent, **kwargs) self.fixture = Fixture(self._path) - def collect(self) -> Iterable[Union[pytest.Item, pytest.Collector]]: + def collect(self) -> Iterable[pytest.Item | pytest.Collector]: """Return a list of children (items and collectors) for this collection node.""" if self.fixture.exception_path.exists(): @@ -71,7 +73,7 @@ def collect(self) -> Iterable[Union[pytest.Item, pytest.Collector]]: WebPoetItem.from_parent(parent=self, name="item", fixture=self.fixture) ] else: - overall_tests: List[pytest.Item] = [ + overall_tests: list[pytest.Item] = [ WebPoetNoToItemException.from_parent( parent=self, name="TO_ITEM_DOESNT_RAISE", fixture=self.fixture ), @@ -79,7 +81,7 @@ def collect(self) -> Iterable[Union[pytest.Item, pytest.Collector]]: parent=self, name="NO_EXTRA_FIELDS", fixture=self.fixture ), ] - field_tests: List[pytest.Item] = [ + field_tests: list[pytest.Item] = [ WebPoetFieldItem.from_parent( parent=self, name=field, fixture=self.fixture, field_name=field ) @@ -222,12 +224,12 @@ def repr_failure(self, excinfo, style=None): return super().repr_failure(excinfo, style) -_found_type_dirs: Set[Path] = set() +_found_type_dirs: set[Path] = set() def collect_file_hook( file_path: Path, parent: pytest.Collector -) -> Optional[pytest.Collector]: +) -> pytest.Collector | None: if file_path.name in {OUTPUT_FILE_NAME, EXCEPTION_FILE_NAME}: testcase_dir = file_path.parent type_dir = testcase_dir.parent @@ -245,9 +247,7 @@ def collect_file_hook( return None -def pytest_addoption( - parser: "pytest.Parser", pluginmanager: "pytest.PytestPluginManager" -): +def pytest_addoption(parser: pytest.Parser, pluginmanager: pytest.PytestPluginManager): parser.addoption( "--web-poet-test-per-item", dest="WEB_POET_TEST_PER_ITEM", @@ -282,7 +282,7 @@ def _get_file(parent: pytest.Collector, *, path: Path) -> WebPoetFile: def pytest_collect_file( file_path: Path, parent: pytest.Collector - ) -> Optional[pytest.Collector]: + ) -> pytest.Collector | None: return collect_file_hook(file_path, parent) else: @@ -311,5 +311,5 @@ def _get_file(parent: pytest.Collector, *, path: Path) -> WebPoetFile: def pytest_collect_file( # type: ignore[misc] path: py.path.local, parent: pytest.Collector - ) -> Optional[pytest.Collector]: + ) -> pytest.Collector | None: return collect_file_hook(Path(path), parent) diff --git a/web_poet/testing/utils.py b/web_poet/testing/utils.py index ad195c00..623b1223 100644 --- a/web_poet/testing/utils.py +++ b/web_poet/testing/utils.py @@ -1,9 +1,11 @@ +from __future__ import annotations + import pytest from _pytest.assertion.util import assertrepr_compare def comparison_error_message( - config: "pytest.Config", op: str, expected, got, prefix: str = "" + config: pytest.Config, op: str, expected, got, prefix: str = "" ) -> str: """Generate an error message""" lines = [prefix] if prefix else [] diff --git a/web_poet/utils.py b/web_poet/utils.py index 49d7b8c0..a5d6d9dc 100644 --- a/web_poet/utils.py +++ b/web_poet/utils.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import inspect import weakref from collections import deque from collections.abc import Iterable from functools import lru_cache, partial, wraps from types import MethodType -from typing import Any, Callable, List, Optional, Tuple, TypeVar, Union, get_args +from typing import Any, Callable, TypeVar, get_args from warnings import warn import packaging.version @@ -35,7 +37,7 @@ def get_fq_class_name(cls: type) -> str: return f"{cls.__module__}.{cls.__qualname__}" -def _clspath(cls: type, forced: Optional[str] = None) -> str: +def _clspath(cls: type, forced: str | None = None) -> str: if forced is not None: return forced return get_fq_class_name(cls) @@ -233,7 +235,7 @@ async def inner(self, *args, **kwargs): _alru_cache = partial(alru_cache, cache_exceptions=False) -def as_list(value: Optional[Any]) -> List[Any]: +def as_list(value: Any | None) -> list[Any]: """Normalizes the value input as a list. >>> as_list(None) @@ -270,15 +272,13 @@ async def ensure_awaitable(obj): return obj -def str_to_pattern(url_pattern: Union[str, Patterns]) -> Patterns: +def str_to_pattern(url_pattern: str | Patterns) -> Patterns: if isinstance(url_pattern, Patterns): return url_pattern return Patterns([url_pattern]) -def get_generic_param( - cls: type, expected: Union[type, Tuple[type, ...]] -) -> Optional[type]: +def get_generic_param(cls: type, expected: type | tuple[type, ...]) -> type | None: """Search the base classes recursively breadth-first for a generic class and return its param. Returns the param of the first found class that is a subclass of ``expected``. From 7d112db89446cf185b0781c82d80836a5d309773 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Feb 2025 22:03:42 +0500 Subject: [PATCH 2/4] Use CODECOV_TOKEN. --- .github/workflows/tests-ubuntu.yml | 5 ++--- .github/workflows/tests-windows.yml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 95181dcd..f6307a6e 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -38,10 +38,9 @@ jobs: run: | tox - name: coverage - uses: codecov/codecov-action@v4 - if: ${{ success() }} + uses: codecov/codecov-action@v5 with: - fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} check: runs-on: ubuntu-latest diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index da0d340c..8b28713f 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -38,7 +38,6 @@ jobs: run: | tox - name: coverage - uses: codecov/codecov-action@v4 - if: ${{ success() }} + uses: codecov/codecov-action@v5 with: - fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} From 8dcfb95933b08b136284061b8dc4910f5c9b3949 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Feb 2025 22:05:29 +0500 Subject: [PATCH 3/4] Try running tests on Python 3.14. --- .github/workflows/tests-ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 95181dcd..4b820100 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14.0-alpha.4'] env: - TOXENV: py include: From 0f4da748e60d1602ffeff701c7359e2ece40721f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Feb 2025 22:07:59 +0500 Subject: [PATCH 4/4] Help with building lxml on 3.14. --- .github/workflows/tests-ubuntu.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 4b820100..08c5e6c7 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -33,6 +33,11 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install tox + - name: Install -dev for 3.14 + if: contains(matrix.python-version, '3.14') + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxslt-dev - name: tox env: ${{ matrix.env }} run: |