Release 2.12 907 (#195)

2.12.907 (2024-01-12)
=====================

- Fixed our thread safety protection against the experimental
freethreaded Python build.
As expected, the absence of GIL challenged our implementation of
``TrafficPolice`` and
took it to its knees. We reviewed the in-depht logic and improved it for
maximum resilience
and performance. We backported some improvements in
``AsyncTrafficPolice`` when applicable.
- Improved error message whenever the pool capacity have been exhausted.
- Fixed background discrete watcher that never reached some connections
in the pool.
- Bumped allowed upper bound for ``python-socks`` to 2.6.1 (we will have
to manually increase the upper bound
each minor/patch version due to our complex integration that invoke
private classes/APIs)

.github/codeql.yml

@@ -1,3 +1,2 @@
 paths:
 - "src/"
-

.github/dependabot.yml

@@ -3,9 +3,14 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
-      # Ignore all patch releases as we can manually
-      # upgrade if we run into a bug and need a fix.
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    ignore:
+      - dependency-name: "*"
+        update-types: [ "version-update:semver-patch" ]

.github/workflows/integration.yml

@@ -31,7 +31,7 @@ jobs:
 
       - name: "Undo Docker Config: docker-py"
         if: matrix.downstream == 'docker'
-        run: | 
+        run: |
           docker logout
           rm -rf ~/.docker
 

.github/workflows/scorecards.yml

@@ -18,7 +18,7 @@ jobs:
       id-token: write
       contents: read
       actions: read
-    
+
     steps:
       - name: "Checkout repository"
         uses: "actions/checkout@d632683dd7b4114ad314bca15554477dd762a938"

.pre-commit-config.yaml

@@ -1,23 +1,18 @@
+exclude: 'docs/|ci/'
 repos:
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
     hooks:
-      - id: pyupgrade
-        args: ["--py37-plus"]
-
-  - repo: https://github.com/psf/black
-    rev: 23.1.0
+      - id: check-yaml
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.9.1
     hooks:
-      - id: black
-        args: ["--target-version", "py37"]
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
-    hooks:
-      - id: flake8
-        additional_dependencies: [flake8-2020]
+      # Run the linter.
+      - id: ruff
+        args: [ --fix, --target-version=py37 ]
+      # Run the formatter.
+      - id: ruff-format

CHANGES.rst

@@ -1,3 +1,15 @@
+2.12.907 (2024-01-12)
+=====================
+
+- Fixed our thread safety protection against the experimental freethreaded Python build.
+  As expected, the absence of GIL challenged our implementation of ``TrafficPolice`` and
+  took it to its knees. We reviewed the in-depht logic and improved it for maximum resilience
+  and performance. We backported some improvements in ``AsyncTrafficPolice`` when applicable.
+- Improved error message whenever the pool capacity have been exhausted.
+- Fixed background discrete watcher that never reached some connections in the pool.
+- Bumped allowed upper bound for ``python-socks`` to 2.6.1 (we will have to manually increase the upper bound
+  each minor/patch version due to our complex integration that invoke private classes/APIs)
+
 2.12.906 (2024-01-03)
 =====================
 

Dockerfile

@@ -14,6 +14,6 @@ COPY ./test test/
 COPY ./dummyserver dummyserver/
 COPY ./ci ci/
 
-COPY noxfile.py LICENSE.txt pyproject.toml README.md setup.cfg hatch_build.py dev-requirements.txt mypy-requirements.txt .coveragerc ./
+COPY noxfile.py LICENSE.txt pyproject.toml README.md hatch_build.py dev-requirements.txt mypy-requirements.txt .coveragerc ./
 
 CMD nox -s test-3.13 && python -m coverage combine && coverage report --ignore-errors --show-missing --fail-under=80

LibreSSL.Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.7.1-alpine3.8
 
-RUN apk add build-base libffi-dev
+RUN apk add build-base libffi-dev linux-headers
 
 WORKDIR /app
 
@@ -16,6 +16,6 @@ COPY ./test test/
 COPY ./dummyserver dummyserver/
 COPY ./ci ci/
 
-COPY noxfile.py LICENSE.txt pyproject.toml README.md setup.cfg hatch_build.py dev-requirements.txt mypy-requirements.txt ./
+COPY noxfile.py LICENSE.txt pyproject.toml README.md hatch_build.py dev-requirements.txt mypy-requirements.txt ./
 
 CMD nox -s test-3.7

OpenSSL.Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.7.2-alpine3.7
 
-RUN apk add build-base libffi-dev
+RUN apk add build-base libffi-dev linux-headers
 
 WORKDIR /app
 
@@ -16,6 +16,6 @@ COPY ./test test/
 COPY ./dummyserver dummyserver/
 COPY ./ci ci/
 
-COPY noxfile.py LICENSE.txt pyproject.toml README.md setup.cfg hatch_build.py dev-requirements.txt mypy-requirements.txt ./
+COPY noxfile.py LICENSE.txt pyproject.toml README.md hatch_build.py dev-requirements.txt mypy-requirements.txt ./
 
 CMD nox -s test-3.7

README.md

@@ -6,7 +6,7 @@
   <a href="https://pypi.org/project/urllib3-future"><img alt="PyPI Version" src="https://img.shields.io/pypi/v/urllib3-future.svg?maxAge=86400" /></a>
   <a href="https://pypi.org/project/urllib3-future"><img alt="Python Versions" src="https://img.shields.io/pypi/pyversions/urllib3-future.svg?maxAge=86400" /></a>
   <br><small>urllib3.future is as BoringSSL is to OpenSSL but to urllib3 (except support is available!)</small>
-  <br><small>✨🍰 Enjoy HTTP like its 2024 🍰✨</small>
+  <br><small>✨🍰 Enjoy HTTP like its 2025 🍰✨</small>
   <br><small>💰 Promotional offer, get everything and more for <del>40k</del> <b>0</b>$!</small>
   <br><small>Wondering why and how this fork exist? Why urllib3 is stuck? <a href="https://medium.com/@ahmed.tahri/revived-the-promise-made-six-years-ago-for-requests-3-37b440e6a064">Take a peek at this article!</a></small>
 </p>
@@ -37,6 +37,7 @@
 - Post-Quantum Security with QUIC.
 - Detailed connection inspection.
 - HTTP/2 with prior knowledge.
+- Support for free-threaded.
 - Server Side Event (SSE).
 - Multiplexed connection.
 - Mirrored Sync & Async.
@@ -83,7 +84,7 @@ urllib3.future can be installed with [pip](https://pip.pypa.io):
 python -m pip install urllib3.future
 ```
 
-You either do 
+You either do
 
 ```python
 import urllib3
@@ -114,7 +115,7 @@ require `urllib3`.
 
 - **It's a fork**
 
-⚠️ Installing urllib3.future shadows the actual urllib3 package (_depending on installation order_). 
+⚠️ Installing urllib3.future shadows the actual urllib3 package (_depending on installation order_).
 The semver will always be like _MAJOR.MINOR.9PP_ like 2.0.941, the patch node is always greater or equal to 900.
 
 Support for bugs or improvements is served in this repository. We regularly sync this fork
@@ -169,7 +170,7 @@ Here are some of the reasons (not exhaustive) we choose to work this way:
 1. It is faster than its counterpart, we measured gain up to 2X faster in a multithreaded environment using a http2 endpoint.
 2. It works well with gevent / does not conflict. We do not use the standard queue class from stdlib as it does not fit http2+ constraints.
 3. Leveraging recent protocols like http2 and http3 transparently. Code and behaviors does not change one bit.
-4. You do not depend on the standard library to emit http/1 requests, and that is actually a good news. http.client 
+4. You do not depend on the standard library to emit http/1 requests, and that is actually a good news. http.client
   has numerous known flaws but cannot be fixed as we speak. (e.g. urllib3 is based on http.client)
 5. There a ton of other improvement you may leverage, but for that you will need to migrate to Niquests or update your code
   to enable specific capabilities, like but not limited to: "DNS over QUIC, HTTP" / "Happy Eyeballs" / "Native Asyncio" / "Advanced Multiplexing".
@@ -289,7 +290,7 @@ python -m pip install requests
 python -m pip install urllib3.future
 ```
 
-Nowadays, we suggest using the package [**Niquests**](https://github.com/jawah/niquests) as a drop-in replacement for **Requests**. 
+Nowadays, we suggest using the package [**Niquests**](https://github.com/jawah/niquests) as a drop-in replacement for **Requests**.
 It leverages urllib3.future capabilities appropriately.
 
 ## Testing

dev-requirements.txt

@@ -6,7 +6,7 @@ pytest>=7.4.4,<9
 pytest-timeout>=2.3.1,<3
 trustme>=0.9.0,<2
 cryptography<40.0.0; python_version <= '3.8'
-cryptography<44; python_version > '3.8'
+cryptography<45; python_version > '3.8'
 backports.zoneinfo==0.2.1; python_version<"3.9"
 tzdata==2024.2; python_version<"3.8"
 towncrier==21.9.0

docker-compose.yaml

@@ -69,7 +69,7 @@ services:
 
   # haproxy is one of the very few
   # capable of handling RFC8441 natively.
-  # todo: wait for Traefik to implement RFC8441, Caddy is almost ready (v2.9).
+  # todo: wait for Traefik to implement RFC8441, Caddy is ready (v2.9).
   #       golang stdlib is ready for it.
   haproxy:
     image: haproxy:3.1-alpine

dummyserver/testcase.py

@@ -321,7 +321,7 @@ def mark(
         orig_request = HTTPConnection.request
 
         def call_and_mark(
-            target: typing.Callable[..., None]
+            target: typing.Callable[..., None],
         ) -> typing.Callable[..., None]:
             def part(
                 self: HTTPConnection, *args: typing.Any, **kwargs: typing.Any

mypy-requirements.txt

@@ -1,7 +1,7 @@
 mypy==1.13.0; python_version >= '3.8'
 mypy==1.4.1; python_version < '3.8'
 idna>=2.0.0
-cryptography==42.0.5
+cryptography<45
 tornado>=6.1
 pytest>=6.2
 trustme==0.9.0

pyproject.toml

@@ -55,7 +55,7 @@ zstd = [
 ]
 secure = []
 socks = [
-  "python-socks>=2.0,<=2.5.3",
+  "python-socks>=2.0,<=2.6.1",
 ]
 qh3 = [
   "qh3>=1.2.0,<2.0.0",
@@ -131,9 +131,14 @@ filterwarnings = [
     '''ignore:loop is closed:ResourceWarning''',
 ]
 
-[tool.isort]
-profile = "black"
-add_imports = "from __future__ import annotations"
+[tool.ruff.lint]
+ignore = ["E501", "E203", "E721"]
+
+[tool.ruff.lint.per-file-ignores]
+"src/urllib3/contrib/socks.py" = ["F811"]
+
+[tool.ruff.lint.isort]
+required-imports = ["from __future__ import annotations"]
 
 [tool.mypy]
 mypy_path = "src"

src/urllib3/_async/connectionpool.py

@@ -146,6 +146,10 @@ async def close(self) -> None:
         Close all pooled connections and disable the pool.
         """
 
+    @property
+    def is_idle(self) -> bool:
+        raise NotImplementedError
+
 
 # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
 _blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
@@ -284,9 +288,9 @@ class AsyncHTTPConnectionPool(AsyncConnectionPool, AsyncRequestMethods):
     """
 
     scheme = "http"
-    ConnectionCls: (
-        type[AsyncHTTPConnection] | type[AsyncHTTPSConnection]
-    ) = AsyncHTTPConnection
+    ConnectionCls: type[AsyncHTTPConnection] | type[AsyncHTTPSConnection] = (
+        AsyncHTTPConnection
+    )
 
     def __init__(
         self,
@@ -1112,8 +1116,7 @@ async def _make_request(
         extension: AsyncExtensionFromHTTP | None = ...,
         *,
         multiplexed: Literal[True],
-    ) -> ResponsePromise:
-        ...
+    ) -> ResponsePromise: ...
 
     @typing.overload
     async def _make_request(
@@ -1140,8 +1143,7 @@ async def _make_request(
         extension: AsyncExtensionFromHTTP | None = ...,
         *,
         multiplexed: Literal[False] = ...,
-    ) -> AsyncHTTPResponse:
-        ...
+    ) -> AsyncHTTPResponse: ...
 
     async def _make_request(
         self,
@@ -1470,8 +1472,7 @@ async def urlopen(
         *,
         multiplexed: Literal[False] = ...,
         **response_kw: typing.Any,
-    ) -> AsyncHTTPResponse:
-        ...
+    ) -> AsyncHTTPResponse: ...
 
     @typing.overload
     async def urlopen(
@@ -1501,8 +1502,7 @@ async def urlopen(
         *,
         multiplexed: Literal[True],
         **response_kw: typing.Any,
-    ) -> ResponsePromise:
-        ...
+    ) -> ResponsePromise: ...
 
     async def urlopen(
         self,
@@ -1661,7 +1661,9 @@ async def urlopen(
             retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
 
         if release_conn is None:
-            release_conn = preload_content
+            # we want to release the connection by default
+            # each and every time. TrafficPolice brings safety.
+            release_conn = True
 
         # Check host
         if assert_same_host and not self.is_same_host(url):
@@ -1777,7 +1779,6 @@ async def urlopen(
                         "body_pos": body_pos,
                     }
                 )
-                release_this_conn = True if not conn.is_saturated else False
 
             # Everything went great!
             clean_exit = True
@@ -1845,23 +1846,23 @@ async def urlopen(
                     await conn.close()
                     conn = None
                 release_this_conn = True
-            elif conn and conn.is_multiplexed is True:
-                # multiplexing allows us to issue more requests.
-                release_this_conn = True
 
-            if release_this_conn is True and conn is not None:
-                # Put the connection back to be reused. If the connection is
-                # expired then it will be None, which will get replaced with a
-                # fresh connection during _get_conn.
-                await self._put_conn(conn)
-                if (
-                    clean_exit
-                    and isinstance(response, ResponsePromise)
-                    and self.pool is not None
-                ):
-                    self.pool.memorize(response, conn)
-            elif release_this_conn is True and self.pool is not None:
-                await self.pool.kill_cursor()
+            if (
+                clean_exit is True
+                and conn is not None
+                and isinstance(response, ResponsePromise) is True
+            ):
+                self.pool.memorize(response, conn)
+
+            if release_this_conn is True:
+                if conn is not None:
+                    # Put the connection back to be reused. If the connection is
+                    # expired then it will be None, which will get replaced with a
+                    # fresh connection during _get_conn.
+                    if self.pool.is_held(conn) is True:
+                        await self._put_conn(conn)
+                else:
+                    await self.pool.kill_cursor()
 
         if not conn:
             # Try again