Use pre-commit (#64)

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.13.1
+    hooks:
+    - id: isort
+-   repo: https://github.com/psf/black
+    rev: 24.3.0
+    hooks:
+    - id: black
+-   repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+    - id: flake8
+- repo: https://github.com/adamchainz/blacken-docs
+  rev: 1.16.0
+  hooks:
+  - id: blacken-docs
+    additional_dependencies:
+    - black==24.3.0

docs/asyncio_api.rst

@@ -14,9 +14,11 @@ You can use the method ``request_raw`` to perform individual requests:
 
     client = AsyncZyteAPI(api_key="YOUR_API_KEY")
 
+
     async def single_request(url):
         return await client.get({"url": url, "browserHtml": True})
 
+
     response = asyncio.run(single_request("https://books.toscrape.com"))
 
 .. tip:: You can skip the ``api_key`` parameter if you :ref:`use an environment
@@ -34,12 +36,10 @@ parallel, using multiple connections:
     from zyte_api import AsyncZyteAPI, create_session
     from zyte_api.aio.errors import RequestError
 
+
     async def extract_from(urls, n_conn):
         client = AsyncZyteAPI(n_conn=n_conn)
-        requests = [
-            {"url": url, "browserHtml": True}
-            for url in urls
-        ]
+        requests = [{"url": url, "browserHtml": True} for url in urls]
         async with create_session(n_conn) as session:
             res_iter = client.iter(requests, session=session)
             for fut in res_iter:
@@ -51,6 +51,7 @@ parallel, using multiple connections:
                     print(e, file=sys.stderr)
                     raise
 
+
     urls = ["https://toscrape.com", "https://books.toscrape.com"]
     asyncio.run(extract_from(urls, n_conn=15))
 

docs/conf.py

@@ -12,19 +12,22 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../'))
+
+import sphinx_rtd_theme
+
+sys.path.insert(0, os.path.abspath("../"))
 
 
 # -- Project information -----------------------------------------------------
 
-project = u'python-zyte-api'
-copyright = u'2021, Zyte Group Ltd'
-author = u'Zyte Group Ltd'
+project = "python-zyte-api"
+copyright = "2021, Zyte Group Ltd"
+author = "Zyte Group Ltd"
 
 # The short X.Y version
-version = u''
+version = ""
 # The full version, including alpha/beta/rc tags
-release = u'0.4.8'
+release = "0.4.8"
 
 
 # -- General configuration ---------------------------------------------------
@@ -37,37 +40,37 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.githubpages',
-    'sphinx.ext.autosummary',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.githubpages",
+    "sphinx.ext.autosummary",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = 'en'
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = None
@@ -78,12 +81,11 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom themes here, relative to this directory.
 # Add path to the RTD explicitly to robustify builds (otherwise might
 # fail in a clean Debian build env)
-import sphinx_rtd_theme
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
@@ -111,7 +113,7 @@
 # -- Options for HTMLHelp output ---------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'python-zyte-apidoc'
+htmlhelp_basename = "python-zyte-apidoc"
 
 
 # -- Options for LaTeX output ------------------------------------------------
@@ -120,15 +122,12 @@
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -138,8 +137,13 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'python-zyte-api.tex', u'python-zyte-api Documentation',
-     u'Zyte Group Ltd', 'manual'),
+    (
+        master_doc,
+        "python-zyte-api.tex",
+        "python-zyte-api Documentation",
+        "Zyte Group Ltd",
+        "manual",
+    ),
 ]
 
 
@@ -148,8 +152,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'python-zyte-api', u'python-zyte-api Documentation',
-     [author], 1)
+    (master_doc, "python-zyte-api", "python-zyte-api Documentation", [author], 1)
 ]
 
 
@@ -159,9 +162,15 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'python-zyte-api', u'python-zyte-api Documentation',
-     author, 'python-zyte-api', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "python-zyte-api",
+        "python-zyte-api Documentation",
+        author,
+        "python-zyte-api",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 
@@ -180,22 +189,31 @@
 # epub_uid = ''
 
 # A list of files that should not be packed into the epub file.
-epub_exclude_files = ['search.html']
+epub_exclude_files = ["search.html"]
 
 
 # -- Extension configuration -------------------------------------------------
 
 # -- Options for intersphinx extension ---------------------------------------
 intersphinx_mapping = {
-    'python': ('https://docs.python.org/3', None, ),
-    'aiohttp': ('https://docs.aiohttp.org/en/stable/', None, ),
-    'tenacity': ('https://tenacity.readthedocs.io/en/latest/', None, ),
+    "python": (
+        "https://docs.python.org/3",
+        None,
+    ),
+    "aiohttp": (
+        "https://docs.aiohttp.org/en/stable/",
+        None,
+    ),
+    "tenacity": (
+        "https://tenacity.readthedocs.io/en/latest/",
+        None,
+    ),
 }
 
 autodoc_default_options = {
     # 'special-members': '__init__,__call__',
     # 'undoc-members': True,
-    'exclude-members': '__weakref__'
+    "exclude-members": "__weakref__"
 }
 
 add_module_names = False

pyproject.toml

@@ -0,0 +1,6 @@
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+
+[tool.black]
+target-version = ["py38", "py39", "py310", "py311", "py312"]

setup.cfg

@@ -0,0 +1,12 @@
+[flake8]
+ignore =
+    # Style issues handled by black.
+    E501,
+    E203,
+    W503,
+
+per-file-ignores =
+    # F401: Ignore "imported but unused" errors in __init__ files, as those
+    # imports are there to expose submodule functions so they can be imported
+    # directly from that module
+    zyte_api/__init__.py:F401

setup.py

@@ -1,48 +1,49 @@
 #!/usr/bin/env python
 import os
-from setuptools import setup, find_packages
+
+from setuptools import find_packages, setup
 
 
 def get_version():
     about = {}
     here = os.path.abspath(os.path.dirname(__file__))
-    with open(os.path.join(here, 'zyte_api/__version__.py')) as f:
+    with open(os.path.join(here, "zyte_api/__version__.py")) as f:
         exec(f.read(), about)
-    return about['__version__']
+    return about["__version__"]
 
 
 setup(
-    name='zyte-api',
+    name="zyte-api",
     version=get_version(),
-    description='Python interface to Zyte API',
-    long_description=open('README.rst').read() + "\n\n" + open('CHANGES.rst').read(),
-    long_description_content_type='text/x-rst',
-    author='Zyte Group Ltd',
-    author_email='[email protected]',
-    url='https://github.com/zytedata/python-zyte-api',
-    packages=find_packages(exclude=['tests', 'examples']),
-    entry_points = {
-        'console_scripts': ['zyte-api=zyte_api.__main__:_main'],
+    description="Python interface to Zyte API",
+    long_description=open("README.rst").read() + "\n\n" + open("CHANGES.rst").read(),
+    long_description_content_type="text/x-rst",
+    author="Zyte Group Ltd",
+    author_email="[email protected]",
+    url="https://github.com/zytedata/python-zyte-api",
+    packages=find_packages(exclude=["tests", "examples"]),
+    entry_points={
+        "console_scripts": ["zyte-api=zyte_api.__main__:_main"],
     },
     install_requires=[
-        'aiohttp >= 3.8.0',
-        'attrs',
-        'brotli',
-        'runstats',
-        'tenacity',
-        'tqdm',
-        'w3lib >= 2.1.1',
+        "aiohttp >= 3.8.0",
+        "attrs",
+        "brotli",
+        "runstats",
+        "tenacity",
+        "tqdm",
+        "w3lib >= 2.1.1",
     ],
     classifiers=[
-        'Development Status :: 3 - Alpha',
-        'Intended Audience :: Developers',
-        'License :: OSI Approved :: BSD License',
-        'Natural Language :: English',
-        'Operating System :: OS Independent',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: BSD License",
+        "Natural Language :: English",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
 )

tests/mockserver.py

@@ -4,18 +4,14 @@
 import sys
 import time
 from base64 import b64encode
-from contextlib import asynccontextmanager
 from importlib import import_module
 from subprocess import PIPE, Popen
-from typing import Any, Dict, Optional
+from typing import Any, Dict
 from urllib.parse import urlparse
 
-from pytest_twisted import ensureDeferred
 from twisted.internet import reactor
-from twisted.internet.defer import Deferred
-from twisted.internet.task import deferLater
 from twisted.web.resource import Resource
-from twisted.web.server import NOT_DONE_YET, Site
+from twisted.web.server import Site
 
 
 def get_ephemeral_port():