From 3919b0f20e4e6ed98fb61fe8e69b5d995dc00abe Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Sun, 19 Nov 2023 16:18:55 +0100 Subject: [PATCH] :bookmark: Release 3.3.2 **Fixed** - Hooks that does not accept keyword arguments are rejected. - Applying `max_fetch` to `Session.gather(...)` did not prevent the adapter to drain all pending responses. - Closed session having unconsumed multiplexed requests leaked an exception from urllib3.future. **Changed** - Aligned `qh3` version constraint in `http3` extra with urllib3.future. --- HISTORY.md | 11 +++++ README.md | 2 +- docs/user/quickstart.rst | 82 +++++++++++++++++++++++++++++-------- pyproject.toml | 2 +- src/niquests/__version__.py | 4 +- src/niquests/adapters.py | 13 ++++-- src/niquests/hooks.py | 5 ++- tests/test_hooks.py | 16 ++++++++ tests/test_multiplexed.py | 32 +++++++++++++++ 9 files changed, 142 insertions(+), 25 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index fdc56fb6d5..4568a3ee1a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,17 @@ Release History =============== +3.3.2 (2023-11-19) +------------------ + +**Fixed** +- Hooks that does not accept keyword arguments are rejected. +- Applying `max_fetch` to `Session.gather(...)` did not prevent the adapter to drain all pending responses. +- Closed session having unconsumed multiplexed requests leaked an exception from urllib3.future. + +**Changed** +- Aligned `qh3` version constraint in `http3` extra with urllib3.future. + 3.3.1 (2023-11-18) ------------------ diff --git a/README.md b/README.md index 477f2b8273..6ab9912072 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ feature freeze. Niquests, is the “**Safest**, **Fastest***, **Easiest**, and **Most advanced**” Python HTTP Client. ✔️ **Try before you switch:** [See Multiplexed in Action](https://replit.com/@ahmedtahri4/Python#main.py)
-📖 **See why you should switch:** [Read about 10 reasons you should](https://medium.com/dev-genius/10-reasons-you-should-quit-your-http-client-98fd4c94bef3) +📖 **See why you should switch:** [Read about 10 reasons why](https://medium.com/dev-genius/10-reasons-you-should-quit-your-http-client-98fd4c94bef3) ```python >>> import niquests diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index b816e8002e..666e8181d3 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -688,6 +688,24 @@ Any ``Response`` returned by get, post, put, etc... will be a lazy instance of ` The possible algorithms are actually nearly limitless, and you may arrange/write you own scheduling technics! +.. warning:: Beware that all in-flight (unresolved) lazy responses are lost immediately after closing the ``Session``. Trying to access unresolved and lost responses will result in ``MultiplexingError`` exception being raised. + +Session Gather +-------------- + +The ``Session`` instance expose a method called ``gather(*responses, max_fetch = None)``, you may call it to +improve the efficiency of resolving your _lazy_ responses. + +Here are the possible outcome of invocation:: + + s.gather() # resolve all pending "lazy" responses + s.gather(resp) # resolve given "resp" only + s.gather(max_fetch=2) # resolve two responses (the first two that come) + s.gather(resp_a, resp_b, resp_c) # resolve all three + s.gather(resp_a, resp_b, resp_c, max_fetch=1) # only resolve the first one + +.. note:: Call to ``s.gather`` is optional, you can access at will the responses properties and methods at any time. + Async session ------------- @@ -698,38 +716,70 @@ All known methods remain the same at the sole difference that it return a corout .. note:: The underlying main library **urllib3.future** does not support native async but is thread safe. This is why we choose to implement / backport `sync_to_async` from Django that use a ThreadPool under the carpet. -Here is an example:: +Here is a basic example:: - from niquests import AsyncSession import asyncio - from time import time + from niquests import AsyncSession, Response - async def emit() -> None: - responses = [] + async def fetch(url: str) -> Response: + with AsyncSession() as s: + return await s.get(url) - async with AsyncSession() as s: # it also work well using multiplexed=True - responses.append(await s.get("https://pie.dev/get")) - responses.append(await s.get("https://pie.dev/delay/3")) + async def main() -> None: + tasks = [] - await s.gather() + for _ in range(10): + tasks.append(asyncio.create_task(fetch("https://pie.dev/delay/1"))) + + responses = await asyncio.gather(*tasks) print(responses) - async def main() -> None: - foo = asyncio.create_task(emit()) - bar = asyncio.create_task(emit()) - await foo - await bar if __name__ == "__main__": - before = time() asyncio.run(main()) - print(time() - before) # 3s! + .. warning:: For the time being **Niquests** only support **asyncio** as the backend library for async. Contributions are welcomed if you want it to be compatible with **anyio** for example. .. note:: Shortcut functions `get`, `post`, ..., from the top-level package does not support async. +Async and Multiplex +------------------- + +You can leverage a multiplexed connection while in an async context! +It's the perfect solution while dealing with two or more hosts that support HTTP/2 onward. + +Look at this basic sample:: + + import asyncio + from niquests import AsyncSession, Response + + async def fetch(url: str) -> list[Response]: + responses = [] + + with AsyncSession(multiplexed=True) as s: + for _ in range(10): + responses.append(await s.get(url)) + + await s.gather() + + return responses + + async def main() -> None: + tasks = [] + + for _ in range(10): + tasks.append(asyncio.create_task(fetch("https://pie.dev/delay/1"))) + + responses_responses = await asyncio.gather(*tasks) + responses = [item for sublist in responses_responses for item in sublist] + + print(responses) + + if __name__ == "__main__": + asyncio.run(main()) + ----------------------- Ready for more? Check out the :ref:`advanced ` section. diff --git a/pyproject.toml b/pyproject.toml index 20a88a29f3..32354c74f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ socks = [ "PySocks>=1.5.6, !=1.5.7", ] http3 = [ - "qh3<1.0.0,>=0.13.0" + "qh3<1.0.0,>=0.14.0" ] ocsp = [ "cryptography<42.0.0,>=41.0.0" diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index 0271b7eed5..c8f2503122 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.3.1" +__version__ = "3.3.2" -__build__: int = 0x030301 +__build__: int = 0x030302 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" diff --git a/src/niquests/adapters.py b/src/niquests/adapters.py index 56265936fa..7f140a2c6b 100644 --- a/src/niquests/adapters.py +++ b/src/niquests/adapters.py @@ -941,7 +941,7 @@ def gather(self, *responses: Response, max_fetch: int | None = None) -> None: if not responses: while True: if max_fetch is not None and max_fetch == 0: - break + return low_resp = self.poolmanager.get_response() @@ -973,7 +973,7 @@ def gather(self, *responses: Response, max_fetch: int | None = None) -> None: # ...Or we have a list on which we should focus. for response in responses: if max_fetch is not None and max_fetch == 0: - break + return req = response.request @@ -982,11 +982,16 @@ def gather(self, *responses: Response, max_fetch: int | None = None) -> None: if not hasattr(response, "_promise"): continue - low_resp = self.poolmanager.get_response(promise=response._promise) + try: + low_resp = self.poolmanager.get_response( + promise=response._promise + ) + except ValueError: + low_resp = None if low_resp is None: raise MultiplexingError( - "Underlying library did not recognize our promise when asked to retrieve it" + "Underlying library did not recognize our promise when asked to retrieve it. Did you close the session too early?" ) if max_fetch is not None: diff --git a/src/niquests/hooks.py b/src/niquests/hooks.py index 2a11858dc4..df7e231e12 100644 --- a/src/niquests/hooks.py +++ b/src/niquests/hooks.py @@ -45,7 +45,10 @@ def dispatch_hook( if callable(callables): callables = [callables] for hook in callables: - _hook_data = hook(hook_data, **kwargs) + try: + _hook_data = hook(hook_data, **kwargs) + except TypeError: + _hook_data = hook(hook_data) if _hook_data is not None: hook_data = _hook_data diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 8cda93dcd1..a46784ff1b 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -20,5 +20,21 @@ def test_hooks(hooks_list, result): assert hooks.dispatch_hook("response", {"response": hooks_list}, "Data") == result +@pytest.mark.parametrize( + "hooks_list, result", + ( + (hook, "ata"), + ([hook, lambda x: None, hook], "ta"), + ), +) +def test_hooks_with_kwargs(hooks_list, result): + assert ( + hooks.dispatch_hook( + "response", {"response": hooks_list}, "Data", should_not_crash=True + ) + == result + ) + + def test_default_hooks(): assert hooks.default_hooks() == {"pre_request": [], "pre_send": [], "response": []} diff --git a/tests/test_multiplexed.py b/tests/test_multiplexed.py index 2270a794ff..fd35ae5bed 100644 --- a/tests/test_multiplexed.py +++ b/tests/test_multiplexed.py @@ -3,6 +3,7 @@ import pytest from niquests import Session +from niquests.exceptions import MultiplexingError @pytest.mark.usefixtures("requires_wan") @@ -76,3 +77,34 @@ def test_get_stream_with_multiplexed(self): import json assert isinstance(json.loads(payload), dict) + + def test_one_at_a_time(self): + responses = [] + + with Session(multiplexed=True) as s: + for _ in [3, 1, 3, 5]: + responses.append(s.get(f"https://pie.dev/delay/{_}")) + + assert all(r.lazy for r in responses) + promise_count = len(responses) + + while any(r.lazy for r in responses): + s.gather(max_fetch=1) + promise_count -= 1 + + assert len(list(filter(lambda r: r.lazy, responses))) == promise_count + + assert len(list(filter(lambda r: r.lazy, responses))) == 0 + + def test_early_close_error(self): + responses = [] + + with Session(multiplexed=True) as s: + for _ in [2, 1, 1]: + responses.append(s.get(f"https://pie.dev/delay/{_}")) + + assert all(r.lazy for r in responses) + + with pytest.raises(MultiplexingError) as exc: + responses[0].json() + assert "Did you close the session too early?" in exc.value.args[0]