From 48fe2816fb8e4833eea8c118866c84c10e686d00 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Sat, 18 Jan 2020 09:46:54 +0100 Subject: [PATCH 01/22] Add .sleep() to backends --- httpx/backends/asyncio.py | 3 +++ httpx/backends/auto.py | 3 +++ httpx/backends/base.py | 3 +++ httpx/backends/trio.py | 3 +++ 4 files changed, 12 insertions(+) diff --git a/httpx/backends/asyncio.py b/httpx/backends/asyncio.py index 8d1025748b..7371be7908 100644 --- a/httpx/backends/asyncio.py +++ b/httpx/backends/asyncio.py @@ -225,6 +225,9 @@ async def open_uds_stream( return SocketStream(stream_reader=stream_reader, stream_writer=stream_writer) + async def sleep(self, seconds: float) -> None: + await asyncio.sleep(seconds) + def time(self) -> float: loop = asyncio.get_event_loop() return loop.time() diff --git a/httpx/backends/auto.py b/httpx/backends/auto.py index 7a8c597822..935a2804d7 100644 --- a/httpx/backends/auto.py +++ b/httpx/backends/auto.py @@ -41,6 +41,9 @@ async def open_uds_stream( ) -> BaseSocketStream: return await self.backend.open_uds_stream(path, hostname, ssl_context, timeout) + async def sleep(self, seconds: float) -> None: + await self.backend.sleep(seconds) + def time(self) -> float: return self.backend.time() diff --git a/httpx/backends/base.py b/httpx/backends/base.py index 964d09449f..0c01709328 100644 --- a/httpx/backends/base.py +++ b/httpx/backends/base.py @@ -111,6 +111,9 @@ async def open_uds_stream( ) -> BaseSocketStream: raise NotImplementedError() # pragma: no cover + async def sleep(self, seconds: float) -> None: + raise NotImplementedError() # pragma: no cover + def time(self) -> float: raise NotImplementedError() # pragma: no cover diff --git a/httpx/backends/trio.py b/httpx/backends/trio.py index 33e93e9677..e6bf208d63 100644 --- a/httpx/backends/trio.py +++ b/httpx/backends/trio.py @@ -131,6 +131,9 @@ async def open_uds_stream( raise ConnectTimeout() + async def sleep(self, seconds: float) -> None: + await trio.sleep(seconds) + def time(self) -> float: return trio.current_time() From 6b3ae742f23a8d5e6a2f022d7822c797bcb46978 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Sat, 18 Jan 2020 09:48:05 +0100 Subject: [PATCH 02/22] Add docs --- docs/advanced.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/docs/advanced.md b/docs/advanced.md index 1d2e4ca0af..4a26a1c55e 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -466,3 +466,59 @@ If you do need to make HTTPS connections to a local server, for example to test >>> r Response <200 OK> ``` + +## Retries + +Communicating with a peer over a network is by essence subject to errors. HTTPX provides built-in retry functionality to increase the resilience to unexpected issues such as network faults or connection issues. + +The default behavior is to retry at most 3 times on connection and network errors before marking the request as failed and bubbling up any exceptions. The delay between retries is increased each time to prevent overloading the requested server. + +### Setting and disabling retries + +You can set retries for an individual request: + +```python +# Using the top-level API: +httpx.get('https://www.example.org', retries=5) + +# Using a client instance: +with httpx.Client() as client: + client.get("https://www.example.org", retries=5) +``` + +Or disable retries for an individual request: + +```python +# Using the top-level API: +httpx.get('https://www.example.org', retries=None) + +# Using a client instance: +with httpx.Client() as client: + client.get("https://www.example.org", retries=None) +``` + +### Setting default retries on the client + +You can set the retry behavior on a client instance, which results in the given behavior being used as the default for requests made with this client: + +```python +client = httpx.Client() # Default behavior: retry at most 3 times. +client = httpx.Client(retries=5) # Retry at most 5 times. +client = httpx.Client(retries=None) # Disable retries by default. +``` + +### Fine-tuning the retries configuration + +The `retries` argument also accepts an instance of `httpx.Retries()`, in case you need more fine-grained control over the retries behavior. It accepts the following parameters: + +- `limit`: the maximum number of retryable errors to retry on. +- `backoff_factor`: a number representing how fast to increase the retry delay. For example, a value of `0.2` (the default) corresponds to this sequence of delays: `(0s, 0.2s, 0.4s, 0.8s, 1.6s, ...)`. + +```python +import httpx + +# Retry at most 5 times, and space out retries further away +# in time than the default (0s, 1s, 2s, 4s, ...). +retries = httpx.Retries(limit=5, backoff_factor=1.0) +response = httpx.get('https://www.example.com', retries=retries) +``` From 24d45e0d1c4285bf99af6b17c31a18522abf5ad0 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Sat, 18 Jan 2020 12:32:28 +0100 Subject: [PATCH 03/22] Add retries --- httpx/__init__.py | 8 +- httpx/client.py | 68 +++++++++++++++- httpx/config.py | 70 +++++++++++++++- httpx/exceptions.py | 9 +++ httpx/retries.py | 191 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 341 insertions(+), 5 deletions(-) create mode 100644 httpx/retries.py diff --git a/httpx/__init__.py b/httpx/__init__.py index 4a133e8efd..9f2497c563 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -2,7 +2,7 @@ from .api import delete, get, head, options, patch, post, put, request, stream from .auth import Auth, BasicAuth, DigestAuth from .client import AsyncClient, Client -from .config import PoolLimits, Proxy, Timeout +from .config import PoolLimits, Proxy, Retries, Timeout from .dispatch.asgi import ASGIDispatch from .dispatch.wsgi import WSGIDispatch from .exceptions import ( @@ -25,9 +25,11 @@ StreamConsumed, TimeoutException, TooManyRedirects, + TooManyRetries, WriteTimeout, ) from .models import URL, Cookies, Headers, QueryParams, Request, Response +from .retries import RetryLimits, RetryOnConnectionFailures from .status_codes import StatusCode, codes __all__ = [ @@ -54,6 +56,10 @@ "PoolLimits", "Proxy", "Timeout", + "Retries", + "RetryLimits", + "RetryOnConnectionFailures", + "TooManyRetries", "ConnectTimeout", "CookieConflict", "ConnectionClosed", diff --git a/httpx/client.py b/httpx/client.py index 9eec6641bb..bec80df974 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -5,16 +5,19 @@ import hstspreload from .auth import Auth, AuthTypes, BasicAuth, FunctionAuth -from .backends.base import ConcurrencyBackend +from .backends.base import ConcurrencyBackend, lookup_backend from .config import ( DEFAULT_MAX_REDIRECTS, DEFAULT_POOL_LIMITS, + DEFAULT_RETRIES_CONFIG, DEFAULT_TIMEOUT_CONFIG, UNSET, CertTypes, PoolLimits, ProxiesTypes, Proxy, + Retries, + RetriesTypes, Timeout, TimeoutTypes, UnsetType, @@ -33,6 +36,7 @@ RedirectLoop, RequestBodyUnavailable, TooManyRedirects, + TooManyRetries, ) from .models import ( URL, @@ -64,6 +68,7 @@ def __init__( headers: HeaderTypes = None, cookies: CookieTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, max_redirects: int = DEFAULT_MAX_REDIRECTS, base_url: URLTypes = None, trust_env: bool = True, @@ -81,6 +86,7 @@ def __init__( self._headers = Headers(headers) self._cookies = Cookies(cookies) self.timeout = Timeout(timeout) + self.retries = Retries(retries) self.max_redirects = max_redirects self.trust_env = trust_env self.netrc = NetRCInfo() @@ -941,6 +947,7 @@ def __init__( http2: bool = False, proxies: ProxiesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, pool_limits: PoolLimits = DEFAULT_POOL_LIMITS, max_redirects: int = DEFAULT_MAX_REDIRECTS, base_url: URLTypes = None, @@ -956,6 +963,7 @@ def __init__( headers=headers, cookies=cookies, timeout=timeout, + retries=retries, max_redirects=max_redirects, base_url=base_url, trust_env=trust_env, @@ -1106,10 +1114,16 @@ async def send( timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) + retries = self.retries + auth = self.build_auth(request, auth) - response = await self.send_handling_redirects( - request, auth=auth, timeout=timeout, allow_redirects=allow_redirects, + response = await self.send_handling_retries( + request, + auth=auth, + timeout=timeout, + retries=retries, + allow_redirects=allow_redirects, ) if not stream: @@ -1120,6 +1134,54 @@ async def send( return response + async def send_handling_retries( + self, + request: Request, + auth: Auth, + retries: Retries, + timeout: Timeout, + allow_redirects: bool = True, + ) -> Response: + backend = lookup_backend() + + delays = retries.get_delays() + retry_flow = retries.retry_flow(request) + + # Initialize the generators. + next(delays) + request = next(retry_flow) + + while True: + try: + response = await self.send_handling_redirects( + request, + auth=auth, + timeout=timeout, + allow_redirects=allow_redirects, + ) + except HTTPError as exc: + logger.debug(f"HTTP Request failed: {exc!r}") + try: + request = retry_flow.throw(type(exc), exc, exc.__traceback__) + except (TooManyRetries, HTTPError): + raise + else: + delay = next(delays) + logger.debug(f"Retrying in {delay} seconds") + await backend.sleep(delay) + else: + try: + request = retry_flow.send(response) + except TooManyRetries: + raise + except StopIteration: + return response + else: + delay = next(delays) + logger.debug(f"Retrying in {delay} seconds") + await backend.sleep(delay) + continue + async def send_handling_redirects( self, request: Request, diff --git a/httpx/config.py b/httpx/config.py index d31086493c..34ca5c6b46 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -1,3 +1,4 @@ +import itertools import os import ssl import typing @@ -5,7 +6,8 @@ import certifi -from .models import URL, Headers, HeaderTypes, URLTypes +from .models import URL, Headers, HeaderTypes, Request, Response, URLTypes +from .retries import DontRetry, RetryLimits, RetryOnConnectionFailures from .utils import get_ca_bundle_from_env, get_logger CertTypes = typing.Union[str, typing.Tuple[str, str], typing.Tuple[str, str, str]] @@ -16,6 +18,7 @@ ProxiesTypes = typing.Union[ URLTypes, "Proxy", typing.Dict[URLTypes, typing.Union[URLTypes, "Proxy"]] ] +RetriesTypes = typing.Union[int, "RetryLimits", "Retries"] DEFAULT_CIPHERS = ":".join( @@ -337,6 +340,71 @@ def __repr__(self) -> str: ) +class Retries: + """ + Retries configuration. + + Holds a retry limiting policy, and implements a configurable exponential + backoff algorithm. + + **Usage**: + + ```python + httpx.Retries() # Default: at most 3 retries on connection failures. + httpx.Retries(0) # Disable retries. + httpx.Retries(5) # At most 5 retries on connection failures. + httpx.Retries( # at most 3 retries on connection failures, with slower backoff. + 5, backoff_factor=1.0 + ) + # Custom retry limiting policy. + httpx.Retries(RetryOnSomeCondition(...)) + # At most 5 retries on connection failures, custom policy for other errors. + httpx.Retries(httpx.RetryOnConnectionFailures(5) | RetryOnSomeOtherCondition(...)) + ``` + """ + + def __init__( + self, + limits: RetriesTypes = 3, + *, + backoff_factor: typing.Union[float, UnsetType] = UNSET, + ) -> None: + if isinstance(limits, int): + limits = RetryOnConnectionFailures(limits) if limits > 0 else DontRetry() + elif isinstance(limits, Retries): + if isinstance(backoff_factor, UnsetType): + backoff_factor = limits.backoff_factor + limits = limits.limits + else: + assert isinstance(limits, RetryLimits) + + if isinstance(backoff_factor, UnsetType): + backoff_factor = 0.2 + + assert backoff_factor > 0 + self.limits: RetryLimits = limits + self.backoff_factor: float = backoff_factor + + def get_delays(self) -> typing.Iterator[float]: + """ + Used by clients to determine how long to wait before issuing a new request. + """ + yield 0 # Send the initial request. + yield 0 # Retry immediately. + for n in itertools.count(2): + yield self.backoff_factor * (2 ** (n - 2)) + + def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + """ + Used by clients to determine what to do when failing to receive a response, + or when a response was received. + + Delegates to the retry limiting policy. + """ + yield from self.limits.retry_flow(request) + + DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0) +DEFAULT_RETRIES_CONFIG = Retries(3, backoff_factor=0.2) DEFAULT_POOL_LIMITS = PoolLimits(soft_limit=10, hard_limit=100) DEFAULT_MAX_REDIRECTS = 20 diff --git a/httpx/exceptions.py b/httpx/exceptions.py index 7efe6fb3c9..951636b0b7 100644 --- a/httpx/exceptions.py +++ b/httpx/exceptions.py @@ -113,6 +113,15 @@ class NotRedirectResponse(RedirectError): """ +# Retries... + + +class TooManyRetries(HTTPError): + """ + The maximum number of retries allowed for a request was exceeded. + """ + + # Stream exceptions... diff --git a/httpx/retries.py b/httpx/retries.py new file mode 100644 index 0000000000..23ad284798 --- /dev/null +++ b/httpx/retries.py @@ -0,0 +1,191 @@ +import typing + +from .exceptions import ( + ConnectTimeout, + HTTPError, + NetworkError, + PoolTimeout, + TooManyRetries, +) +from .models import Request, Response +from .utils import get_logger + +logger = get_logger(__name__) + + +class RetryLimits: + """ + Base class for retry limiting policies. + """ + + def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + """ + Execute the retry flow. + + To dispatch a request, you should `yield` it, and prepare for the following + situations: + + * The request resulted in an `httpx.HTTPError`. If it should be retried on, + you should make any necessary modifications to the request, and continue + yielding. If you've exceeded the maximum number of retries, wrap the error + in `httpx.TooManyRetries()` and raise the result. If it shouldn't be retried + on, re-`raise` the error as-is. + * The request went through and resulted in the client sending back a `response`. + If it should be retried on (e.g. because it is an error response), you + should make any necessary modifications to the request, and continue yielding. + Otherwise, `return` to terminate the retry flow. + + Note that modifying the request may cause downstream mechanisms that rely + on request signing to fail. For example, this could be the case of + certain authentication schemes. + + A typical pseudo-code implementation based on a while-loop and try/except + blocks may look like this... + + ```python + while True: + try: + response = yield request + except httpx.HTTPError as exc: + if not has_retries_left(): + raise TooManyRetries(exc) + if should_retry_on_exception(exc): + increment_retries_left() + # (Optionally modify the request here.) + continue + else: + raise + else: + if should_retry_on_response(response): + # (Optionally modify the request here.) + continue + return + ``` + """ + raise NotImplementedError + + def __or__(self, other: typing.Any) -> "RetryLimits": + if not isinstance(other, RetryLimits): + raise NotImplementedError + return _OrRetries(self, other) + + +class _OrRetries(RetryLimits): + """ + Helper for composing retry limits. + """ + + def __init__(self, left: RetryLimits, right: RetryLimits) -> None: + self.left = left + self.right = right + + def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + left_flow = self.left.retry_flow(request) + right_flow = self.right.retry_flow(request) + + request = next(left_flow) + request = next(right_flow) + + while True: + try: + response = yield request + except HTTPError as exc: + try: + request = left_flow.throw(type(exc), exc, exc.__traceback__) + except TooManyRetries: + raise + except HTTPError: + try: + request = right_flow.throw(type(exc), exc, exc.__traceback__) + except TooManyRetries: + raise + except HTTPError: + raise + else: + continue + else: + continue + else: + try: + request = left_flow.send(response) + except TooManyRetries: + raise + except StopIteration: + try: + request = right_flow.send(response) + except TooManyRetries: + raise + except StopIteration: + return + else: + continue + else: + continue + + +class DontRetry(RetryLimits): + def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + # Send the initial request, and never retry. + # Don't raise a `TooManyRetries` exception because this should really be + # a no-op implementation. + yield request + + +class RetryOnConnectionFailures(RetryLimits): + """ + Retry when failing to establish a connection, or when a network + error occurred. + """ + + _RETRYABLE_EXCEPTIONS: typing.Sequence[typing.Type[HTTPError]] = ( + ConnectTimeout, + PoolTimeout, + NetworkError, + ) + _RETRYABLE_METHODS: typing.Container[str] = frozenset( + ("HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE") + ) + + def __init__(self, limit: int = 3) -> None: + assert limit >= 0 + self.limit = limit + + def _should_retry_on_exception(self, exc: HTTPError) -> bool: + for exc_cls in self._RETRYABLE_EXCEPTIONS: + if isinstance(exc, exc_cls): + break + else: + logger.debug(f"not_retryable exc_type={type(exc)}") + return False + + assert exc.request is not None + method = exc.request.method.upper() + if method not in self._RETRYABLE_METHODS: + logger.debug(f"not_retryable method={method!r}") + return False + + return True + + def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + retries_left = self.limit + + while True: + try: + _ = yield request + except HTTPError as exc: + # Failed to get a response... + + if not retries_left: + raise TooManyRetries(exc, request=request) + + if self._should_retry_on_exception(exc): + retries_left -= 1 + continue + + # Raise the exception for other retry limits involved to handle, + # or for bubbling up to the client. + raise + else: + # We managed to get a response without connection/network + # failures, so we're done here. + return From 89d04cc1ed7258d9fc9afd7cd8726a84c35eb5f0 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Sat, 18 Jan 2020 13:48:26 +0100 Subject: [PATCH 04/22] Refine docs, interfaces and implementation --- docs/advanced.md | 87 ++++++++++++++++++++++++++++++++--------------- httpx/__init__.py | 3 +- httpx/config.py | 62 +++++++++++++++++++-------------- httpx/retries.py | 66 ++++++++++++++--------------------- 4 files changed, 122 insertions(+), 96 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index 4a26a1c55e..343d05762c 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -475,50 +475,81 @@ The default behavior is to retry at most 3 times on connection and network error ### Setting and disabling retries -You can set retries for an individual request: +You can set the retry behavior on a client instance, which results in the given behavior being used for all requests made with this client: ```python -# Using the top-level API: -httpx.get('https://www.example.org', retries=5) - -# Using a client instance: -with httpx.Client() as client: - client.get("https://www.example.org", retries=5) +client = httpx.Client() # Retry at most 3 times on connection failures. +client = httpx.Client(retries=5) # Retry at most 5 times on connection failures. +client = httpx.Client(retries=0) # Disable retries. ``` -Or disable retries for an individual request: +### Fine-tuning the retries configuration -```python -# Using the top-level API: -httpx.get('https://www.example.org', retries=None) +When instantiating a client, the `retries` argument may be one of the following... -# Using a client instance: -with httpx.Client() as client: - client.get("https://www.example.org", retries=None) -``` +* An integer, representing the maximum number connection failures to retry on. Use `0` to disable retries entirely. -### Setting default retries on the client +```python +client = httpx.Client(retries=5) +``` -You can set the retry behavior on a client instance, which results in the given behavior being used as the default for requests made with this client: +* An `httpx.Retries()` instance. It accepts the number of connection failures to retry on as a positional argument. The `backoff_factor` keyword argument that specifies how fast the time to wait before issuing a retry request should be increased. By default this is `0.2`, which corresponds to issuing a new request after `(0s, 0.2s, 0.4s, 0.8s, ...)`. (Note that a lot of errors are immediately resolved after retrying, so HTTPX will always issue the initial retry right away.) ```python -client = httpx.Client() # Default behavior: retry at most 3 times. -client = httpx.Client(retries=5) # Retry at most 5 times. -client = httpx.Client(retries=None) # Disable retries by default. +# Retry at most 5 times on connection failures, +# and issue new requests after `(0s, 0.5s, 1s, 2s, 4s, ...)` +retries = httpx.Retries(5, backoff_factor=0.5) +client = httpx.Client(retries=retries) ``` -### Fine-tuning the retries configuration +### Advanced retries customization -The `retries` argument also accepts an instance of `httpx.Retries()`, in case you need more fine-grained control over the retries behavior. It accepts the following parameters: +The first argument to `httpx.Retries()` can also be a subclass of `httpx.RetryLimits`. This is useful if you want to replace or extend the default behavior of retrying on connection failures. -- `limit`: the maximum number of retryable errors to retry on. -- `backoff_factor`: a number representing how fast to increase the retry delay. For example, a value of `0.2` (the default) corresponds to this sequence of delays: `(0s, 0.2s, 0.4s, 0.8s, 1.6s, ...)`. +The `httpx.RetryLimits` subclass should implement the `.retry_flow()` method, `yield` any request to be made, and prepare for the following situations... + +* (A) The request resulted in an `httpx.HTTPError`. If it shouldn't be retried on, `raise` the error as-is. If it should be retried on, you should make any necessary modifications to the request, and continue yielding. If you've exceeded a maximum number of retries, wrap the error in `httpx.TooManyRetries()`, and raise the result. +* (B) The request went through and resulted in the client sending back a `response`. If it shouldn't be retried on, `return` to terminate the retry flow. If it should be retried on (e.g. because it is an error response), you should make any necessary modifications to the request, and continue yielding. If you've exceeded a maximum number of retries, wrap the response in `httpx.TooManyRetries()`, and raise the result. + +As an example, here's how you could implement a custom retry limiting policy that retries on certain status codes: ```python import httpx -# Retry at most 5 times, and space out retries further away -# in time than the default (0s, 1s, 2s, 4s, ...). -retries = httpx.Retries(limit=5, backoff_factor=1.0) -response = httpx.get('https://www.example.com', retries=retries) +class RetryOnStatusCodes(httpx.RetryLimits): + def __init__(self, limit, status_codes): + self.limit = limit + self.status_codes = status_codes + + def retry_flow(self, request): + retries_left = self.limit + + while True: + response = yield request + + if response.status_code not in self.status_codes: + return + + if retries_left == 0: + try: + response.raise_for_status() + except httpx.HTTPError as exc: + raise httpx.TooManyRetries(exc, response=response) + else: + raise httpx.TooManyRetries(response=response) + + retries_left -= 1 +``` + +To use a custom policy: + +* Explicitly pass the number of times to retry on connection failures as a first positional argument to `httpx.Retries()`. (Use `0` to not retry on these failures.) +* Pass the custom policy as a second positional argument. + +For example... + +```python +# Retry at most 3 times on connection failures, and at most three times +# on '429 Too Many Requests', '502 Bad Gateway', or '503 Service Unavailable'. +retries = httpx.Retries(3, RetryOnStatusCodes(3, status_codes={429, 502, 503})) ``` diff --git a/httpx/__init__.py b/httpx/__init__.py index 9f2497c563..69d57cb6d9 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -29,7 +29,7 @@ WriteTimeout, ) from .models import URL, Cookies, Headers, QueryParams, Request, Response -from .retries import RetryLimits, RetryOnConnectionFailures +from .retries import RetryLimits from .status_codes import StatusCode, codes __all__ = [ @@ -58,7 +58,6 @@ "Timeout", "Retries", "RetryLimits", - "RetryOnConnectionFailures", "TooManyRetries", "ConnectTimeout", "CookieConflict", diff --git a/httpx/config.py b/httpx/config.py index 34ca5c6b46..06b5dcee7e 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -346,45 +346,57 @@ class Retries: Holds a retry limiting policy, and implements a configurable exponential backoff algorithm. - - **Usage**: - - ```python - httpx.Retries() # Default: at most 3 retries on connection failures. - httpx.Retries(0) # Disable retries. - httpx.Retries(5) # At most 5 retries on connection failures. - httpx.Retries( # at most 3 retries on connection failures, with slower backoff. - 5, backoff_factor=1.0 - ) - # Custom retry limiting policy. - httpx.Retries(RetryOnSomeCondition(...)) - # At most 5 retries on connection failures, custom policy for other errors. - httpx.Retries(httpx.RetryOnConnectionFailures(5) | RetryOnSomeOtherCondition(...)) - ``` """ def __init__( self, - limits: RetriesTypes = 3, - *, - backoff_factor: typing.Union[float, UnsetType] = UNSET, + *retries: RetriesTypes, + backoff_factor: float = None, ) -> None: - if isinstance(limits, int): - limits = RetryOnConnectionFailures(limits) if limits > 0 else DontRetry() - elif isinstance(limits, Retries): - if isinstance(backoff_factor, UnsetType): + limits: RetriesTypes + + if len(retries) == 0: + limits = RetryOnConnectionFailures(3) + elif len(retries) == 1: + limits = retries[0] + if isinstance(limits, int): + limits = ( + RetryOnConnectionFailures(limits) if limits > 0 else DontRetry() + ) + elif isinstance(limits, Retries): + assert backoff_factor is None backoff_factor = limits.backoff_factor - limits = limits.limits + limits = limits.limits + else: + raise NotImplementedError( + "Passing a `RetryLimits` subclass as a single argument " + "is not supported. You must explicitly pass the number of times " + "to retry on connection failures. " + "For example: `Retries(3, MyRetryLimits(...))`." + ) + elif len(retries) == 2: + default, custom = retries + assert isinstance(custom, RetryLimits) + limits = Retries(default).limits | custom else: - assert isinstance(limits, RetryLimits) + raise NotImplementedError( + "Composing more than 2 retry limits is not supported yet." + ) - if isinstance(backoff_factor, UnsetType): + if backoff_factor is None: backoff_factor = 0.2 assert backoff_factor > 0 self.limits: RetryLimits = limits self.backoff_factor: float = backoff_factor + def __eq__(self, other: typing.Any) -> bool: + return ( + isinstance(other, Retries) + and self.limits == other.limits + and self.backoff_factor == other.backoff_factor + ) + def get_delays(self) -> typing.Iterator[float]: """ Used by clients to determine how long to wait before issuing a new request. diff --git a/httpx/retries.py b/httpx/retries.py index 23ad284798..07f0e02c45 100644 --- a/httpx/retries.py +++ b/httpx/retries.py @@ -22,51 +22,20 @@ def retry_flow(self, request: Request) -> typing.Generator[Request, Response, No """ Execute the retry flow. - To dispatch a request, you should `yield` it, and prepare for the following - situations: - - * The request resulted in an `httpx.HTTPError`. If it should be retried on, - you should make any necessary modifications to the request, and continue - yielding. If you've exceeded the maximum number of retries, wrap the error - in `httpx.TooManyRetries()` and raise the result. If it shouldn't be retried - on, re-`raise` the error as-is. - * The request went through and resulted in the client sending back a `response`. - If it should be retried on (e.g. because it is an error response), you - should make any necessary modifications to the request, and continue yielding. - Otherwise, `return` to terminate the retry flow. - - Note that modifying the request may cause downstream mechanisms that rely - on request signing to fail. For example, this could be the case of - certain authentication schemes. - - A typical pseudo-code implementation based on a while-loop and try/except - blocks may look like this... - - ```python - while True: - try: - response = yield request - except httpx.HTTPError as exc: - if not has_retries_left(): - raise TooManyRetries(exc) - if should_retry_on_exception(exc): - increment_retries_left() - # (Optionally modify the request here.) - continue - else: - raise - else: - if should_retry_on_response(response): - # (Optionally modify the request here.) - continue - return - ``` + To dispatch a request, you should `yield` it, and prepare for either + getting a response, or an `HTTPError` being raised. + + In each case, decide whether to retry: + + * If so, continue yielding, unless a maximum number of retries was exceeded. + In that case, raise a `TooManyRetries` exception. + * Otherwise, `return`, or `raise` the exception. """ - raise NotImplementedError + raise NotImplementedError # pragma: no cover def __or__(self, other: typing.Any) -> "RetryLimits": if not isinstance(other, RetryLimits): - raise NotImplementedError + raise NotImplementedError # pragma: no cover return _OrRetries(self, other) @@ -79,6 +48,13 @@ def __init__(self, left: RetryLimits, right: RetryLimits) -> None: self.left = left self.right = right + def __eq__(self, other: typing.Any) -> bool: + return ( + isinstance(other, _OrRetries) + and self.left == other.left + and self.right == other.right + ) + def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: left_flow = self.left.retry_flow(request) right_flow = self.right.retry_flow(request) @@ -124,6 +100,9 @@ def retry_flow(self, request: Request) -> typing.Generator[Request, Response, No class DontRetry(RetryLimits): + def __eq__(self, other: typing.Any) -> bool: + return type(other) == DontRetry + def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: # Send the initial request, and never retry. # Don't raise a `TooManyRetries` exception because this should really be @@ -150,6 +129,11 @@ def __init__(self, limit: int = 3) -> None: assert limit >= 0 self.limit = limit + def __eq__(self, other: typing.Any) -> bool: + return ( + isinstance(other, RetryOnConnectionFailures) and self.limit == other.limit + ) + def _should_retry_on_exception(self, exc: HTTPError) -> bool: for exc_cls in self._RETRYABLE_EXCEPTIONS: if isinstance(exc, exc_cls): From 9d48d7044a11038414338ee6b97a8c2a4133adf6 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Sat, 18 Jan 2020 14:06:47 +0100 Subject: [PATCH 05/22] Lint, fix tests --- httpx/config.py | 6 +----- tests/test_timeouts.py | 6 ++++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/httpx/config.py b/httpx/config.py index 06b5dcee7e..5efe122956 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -348,11 +348,7 @@ class Retries: backoff algorithm. """ - def __init__( - self, - *retries: RetriesTypes, - backoff_factor: float = None, - ) -> None: + def __init__(self, *retries: RetriesTypes, backoff_factor: float = None) -> None: limits: RetriesTypes if len(retries) == 0: diff --git a/tests/test_timeouts.py b/tests/test_timeouts.py index e394e0e301..8754449231 100644 --- a/tests/test_timeouts.py +++ b/tests/test_timeouts.py @@ -26,7 +26,7 @@ async def test_write_timeout(server): async def test_connect_timeout(server): timeout = httpx.Timeout(connect_timeout=1e-6) - async with httpx.AsyncClient(timeout=timeout) as client: + async with httpx.AsyncClient(timeout=timeout, retries=0) as client: with pytest.raises(httpx.ConnectTimeout): # See https://stackoverflow.com/questions/100841/ await client.get("http://10.255.255.1/") @@ -37,7 +37,9 @@ async def test_pool_timeout(server): pool_limits = httpx.PoolLimits(hard_limit=1) timeout = httpx.Timeout(pool_timeout=1e-4) - async with httpx.AsyncClient(pool_limits=pool_limits, timeout=timeout) as client: + async with httpx.AsyncClient( + pool_limits=pool_limits, timeout=timeout, retries=0 + ) as client: async with client.stream("GET", server.url): with pytest.raises(httpx.PoolTimeout): await client.get("http://localhost:8000/") From 6cbce4862a647986b97769697729c7e4790690db Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sun, 19 Jan 2020 08:05:49 +0100 Subject: [PATCH 06/22] Drop custom retry limits --- docs/advanced.md | 54 +------------------------------- httpx/config.py | 48 ++++++++-------------------- httpx/retries.py | 81 ++++++------------------------------------------ 3 files changed, 23 insertions(+), 160 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index 343d05762c..6154bb6ece 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -493,7 +493,7 @@ When instantiating a client, the `retries` argument may be one of the following. client = httpx.Client(retries=5) ``` -* An `httpx.Retries()` instance. It accepts the number of connection failures to retry on as a positional argument. The `backoff_factor` keyword argument that specifies how fast the time to wait before issuing a retry request should be increased. By default this is `0.2`, which corresponds to issuing a new request after `(0s, 0.2s, 0.4s, 0.8s, ...)`. (Note that a lot of errors are immediately resolved after retrying, so HTTPX will always issue the initial retry right away.) +* An `httpx.Retries()` instance. It accepts the number of connection failures to retry on as a positional argument. The `backoff_factor` keyword argument that specifies how fast the time to wait before issuing a retry request should be increased. By default this is `0.2`, which corresponds to issuing a new request after `(0s, 0.2s, 0.4s, 0.8s, ...)`. (Note that a lot of errors are immediately resolved by retrying, so HTTPX will always issue the initial retry right away.) ```python # Retry at most 5 times on connection failures, @@ -501,55 +501,3 @@ client = httpx.Client(retries=5) retries = httpx.Retries(5, backoff_factor=0.5) client = httpx.Client(retries=retries) ``` - -### Advanced retries customization - -The first argument to `httpx.Retries()` can also be a subclass of `httpx.RetryLimits`. This is useful if you want to replace or extend the default behavior of retrying on connection failures. - -The `httpx.RetryLimits` subclass should implement the `.retry_flow()` method, `yield` any request to be made, and prepare for the following situations... - -* (A) The request resulted in an `httpx.HTTPError`. If it shouldn't be retried on, `raise` the error as-is. If it should be retried on, you should make any necessary modifications to the request, and continue yielding. If you've exceeded a maximum number of retries, wrap the error in `httpx.TooManyRetries()`, and raise the result. -* (B) The request went through and resulted in the client sending back a `response`. If it shouldn't be retried on, `return` to terminate the retry flow. If it should be retried on (e.g. because it is an error response), you should make any necessary modifications to the request, and continue yielding. If you've exceeded a maximum number of retries, wrap the response in `httpx.TooManyRetries()`, and raise the result. - -As an example, here's how you could implement a custom retry limiting policy that retries on certain status codes: - -```python -import httpx - -class RetryOnStatusCodes(httpx.RetryLimits): - def __init__(self, limit, status_codes): - self.limit = limit - self.status_codes = status_codes - - def retry_flow(self, request): - retries_left = self.limit - - while True: - response = yield request - - if response.status_code not in self.status_codes: - return - - if retries_left == 0: - try: - response.raise_for_status() - except httpx.HTTPError as exc: - raise httpx.TooManyRetries(exc, response=response) - else: - raise httpx.TooManyRetries(response=response) - - retries_left -= 1 -``` - -To use a custom policy: - -* Explicitly pass the number of times to retry on connection failures as a first positional argument to `httpx.Retries()`. (Use `0` to not retry on these failures.) -* Pass the custom policy as a second positional argument. - -For example... - -```python -# Retry at most 3 times on connection failures, and at most three times -# on '429 Too Many Requests', '502 Bad Gateway', or '503 Service Unavailable'. -retries = httpx.Retries(3, RetryOnStatusCodes(3, status_codes={429, 502, 503})) -``` diff --git a/httpx/config.py b/httpx/config.py index 5efe122956..130b143dae 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -18,7 +18,7 @@ ProxiesTypes = typing.Union[ URLTypes, "Proxy", typing.Dict[URLTypes, typing.Union[URLTypes, "Proxy"]] ] -RetriesTypes = typing.Union[int, "RetryLimits", "Retries"] +RetriesTypes = typing.Union[int, "Retries"] DEFAULT_CIPHERS = ":".join( @@ -344,45 +344,25 @@ class Retries: """ Retries configuration. - Holds a retry limiting policy, and implements a configurable exponential - backoff algorithm. + Defines the retry limiting policy, and implements a configurable + exponential backoff algorithm. """ - def __init__(self, *retries: RetriesTypes, backoff_factor: float = None) -> None: - limits: RetriesTypes - - if len(retries) == 0: - limits = RetryOnConnectionFailures(3) - elif len(retries) == 1: - limits = retries[0] - if isinstance(limits, int): - limits = ( - RetryOnConnectionFailures(limits) if limits > 0 else DontRetry() - ) - elif isinstance(limits, Retries): - assert backoff_factor is None - backoff_factor = limits.backoff_factor - limits = limits.limits - else: - raise NotImplementedError( - "Passing a `RetryLimits` subclass as a single argument " - "is not supported. You must explicitly pass the number of times " - "to retry on connection failures. " - "For example: `Retries(3, MyRetryLimits(...))`." - ) - elif len(retries) == 2: - default, custom = retries - assert isinstance(custom, RetryLimits) - limits = Retries(default).limits | custom - else: - raise NotImplementedError( - "Composing more than 2 retry limits is not supported yet." - ) + def __init__( + self, retries: RetriesTypes = 3, *, backoff_factor: float = None + ) -> None: + if isinstance(retries, int): + limits = RetryOnConnectionFailures(retries) if retries > 0 else DontRetry() + elif isinstance(retries, Retries): + assert backoff_factor is None + backoff_factor = retries.backoff_factor + limits = retries.limits if backoff_factor is None: backoff_factor = 0.2 assert backoff_factor > 0 + self.limits: RetryLimits = limits self.backoff_factor: float = backoff_factor @@ -406,8 +386,6 @@ def retry_flow(self, request: Request) -> typing.Generator[Request, Response, No """ Used by clients to determine what to do when failing to receive a response, or when a response was received. - - Delegates to the retry limiting policy. """ yield from self.limits.retry_flow(request) diff --git a/httpx/retries.py b/httpx/retries.py index 07f0e02c45..f2a4532522 100644 --- a/httpx/retries.py +++ b/httpx/retries.py @@ -22,8 +22,10 @@ def retry_flow(self, request: Request) -> typing.Generator[Request, Response, No """ Execute the retry flow. - To dispatch a request, you should `yield` it, and prepare for either - getting a response, or an `HTTPError` being raised. + To dispatch a request, you should `yield` it, and prepare for either: + + * The client sending back a response. + * An `HTTPError` being raised. In each case, decide whether to retry: @@ -33,71 +35,6 @@ def retry_flow(self, request: Request) -> typing.Generator[Request, Response, No """ raise NotImplementedError # pragma: no cover - def __or__(self, other: typing.Any) -> "RetryLimits": - if not isinstance(other, RetryLimits): - raise NotImplementedError # pragma: no cover - return _OrRetries(self, other) - - -class _OrRetries(RetryLimits): - """ - Helper for composing retry limits. - """ - - def __init__(self, left: RetryLimits, right: RetryLimits) -> None: - self.left = left - self.right = right - - def __eq__(self, other: typing.Any) -> bool: - return ( - isinstance(other, _OrRetries) - and self.left == other.left - and self.right == other.right - ) - - def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: - left_flow = self.left.retry_flow(request) - right_flow = self.right.retry_flow(request) - - request = next(left_flow) - request = next(right_flow) - - while True: - try: - response = yield request - except HTTPError as exc: - try: - request = left_flow.throw(type(exc), exc, exc.__traceback__) - except TooManyRetries: - raise - except HTTPError: - try: - request = right_flow.throw(type(exc), exc, exc.__traceback__) - except TooManyRetries: - raise - except HTTPError: - raise - else: - continue - else: - continue - else: - try: - request = left_flow.send(response) - except TooManyRetries: - raise - except StopIteration: - try: - request = right_flow.send(response) - except TooManyRetries: - raise - except StopIteration: - return - else: - continue - else: - continue - class DontRetry(RetryLimits): def __eq__(self, other: typing.Any) -> bool: @@ -105,8 +42,8 @@ def __eq__(self, other: typing.Any) -> bool: def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: # Send the initial request, and never retry. - # Don't raise a `TooManyRetries` exception because this should really be - # a no-op implementation. + # NOTE: don't raise a `TooManyRetries` exception because this should + # really be a no-op implementation. yield request @@ -134,7 +71,7 @@ def __eq__(self, other: typing.Any) -> bool: isinstance(other, RetryOnConnectionFailures) and self.limit == other.limit ) - def _should_retry_on_exception(self, exc: HTTPError) -> bool: + def _should_retry_for_exception(self, exc: HTTPError) -> bool: for exc_cls in self._RETRYABLE_EXCEPTIONS: if isinstance(exc, exc_cls): break @@ -157,12 +94,12 @@ def retry_flow(self, request: Request) -> typing.Generator[Request, Response, No try: _ = yield request except HTTPError as exc: - # Failed to get a response... + # Failed to get a response. if not retries_left: raise TooManyRetries(exc, request=request) - if self._should_retry_on_exception(exc): + if self._should_retry_for_exception(exc): retries_left -= 1 continue From 6e57f9697dbd508db404ae2b22b1d8a6fd930633 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sun, 19 Jan 2020 08:46:32 +0100 Subject: [PATCH 07/22] Add tests --- httpx/__init__.py | 2 + tests/client/test_retries.py | 144 +++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 tests/client/test_retries.py diff --git a/httpx/__init__.py b/httpx/__init__.py index 69d57cb6d9..6864a40e7c 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -12,6 +12,7 @@ DecodingError, HTTPError, InvalidURL, + NetworkError, NotRedirectResponse, PoolTimeout, ProtocolError, @@ -65,6 +66,7 @@ "DecodingError", "HTTPError", "InvalidURL", + "NetworkError", "NotRedirectResponse", "PoolTimeout", "ProtocolError", diff --git a/tests/client/test_retries.py b/tests/client/test_retries.py new file mode 100644 index 0000000000..ee00f652d6 --- /dev/null +++ b/tests/client/test_retries.py @@ -0,0 +1,144 @@ +import collections +import itertools +import typing + +import pytest + +import httpx +from httpx.config import TimeoutTypes +from httpx.dispatch.base import AsyncDispatcher +from httpx.retries import DontRetry, RetryOnConnectionFailures + + +class MockDispatch(AsyncDispatcher): + _ENDPOINTS: typing.Dict[str, typing.Type[httpx.HTTPError]] = { + "/connect_timeout": httpx.ConnectTimeout, + "/pool_timeout": httpx.PoolTimeout, + "/network_error": httpx.NetworkError, + } + + def __init__(self, succeed_after: int) -> None: + self.succeed_after = succeed_after + self.attempts: typing.DefaultDict[str, int] = collections.defaultdict(int) + + async def send( + self, request: httpx.Request, timeout: TimeoutTypes = None + ) -> httpx.Response: + assert request.url.path in self._ENDPOINTS + + exc_cls = self._ENDPOINTS[request.url.path] + + if self.attempts[request.url.path] < self.succeed_after: + self.attempts[request.url.path] += 1 + raise exc_cls(request=request) + + return httpx.Response(httpx.codes.OK, request=request) + + +@pytest.mark.usefixtures("async_environment") +async def test_no_retries() -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=0) + + with pytest.raises(httpx.ConnectTimeout): + await client.get("https://example.com/connect_timeout") + + with pytest.raises(httpx.PoolTimeout): + await client.get("https://example.com/pool_timeout") + + with pytest.raises(httpx.NetworkError): + await client.get("https://example.com/network_error") + + +@pytest.mark.usefixtures("async_environment") +async def test_default_retries() -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3)) + + response = await client.get("https://example.com/connect_timeout") + assert response.status_code == 200 + + response = await client.get("https://example.com/pool_timeout") + assert response.status_code == 200 + + response = await client.get("https://example.com/network_error") + assert response.status_code == 200 + + +@pytest.mark.usefixtures("async_environment") +async def test_too_many_retries() -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=2), retries=1) + + with pytest.raises(httpx.TooManyRetries): + await client.get("https://example.com/connect_timeout") + + with pytest.raises(httpx.TooManyRetries): + await client.get("https://example.com/pool_timeout") + + with pytest.raises(httpx.TooManyRetries): + await client.get("https://example.com/network_error") + + +@pytest.mark.usefixtures("async_environment") +@pytest.mark.parametrize("method", ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]) +async def test_retries_idempotent_methods(method: str) -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1)) + response = await client.request(method, "https://example.com/connect_timeout") + assert response.status_code == 200 + + +@pytest.mark.usefixtures("async_environment") +async def test_no_retries_non_idempotent_methods() -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1)) + + with pytest.raises(httpx.ConnectTimeout): + await client.post("https://example.com/connect_timeout") + + with pytest.raises(httpx.PoolTimeout): + await client.patch("https://example.com/pool_timeout") + + +@pytest.mark.parametrize( + "retries, delays", + [ + (httpx.Retries(), [0, 0, 0.2, 0.4, 0.8, 1.6]), + (httpx.Retries(backoff_factor=0.1), [0, 0, 0.1, 0.2, 0.4, 0.8]), + ], +) +def test_retries_delays_sequence( + retries: httpx.Retries, delays: typing.List[int] +) -> None: + sample_delays = list(itertools.islice(retries.get_delays(), 6)) + assert sample_delays == delays + + +@pytest.mark.usefixtures("async_environment") +@pytest.mark.parametrize( + "retries, elapsed", + [ + (httpx.Retries(), pytest.approx(0 + 0 + 0.2 + 0.4, rel=0.1)), + (httpx.Retries(backoff_factor=0.1), pytest.approx(0 + 0 + 0.1 + 0.2, rel=0.2)), + ], +) +async def test_retries_backoff(retries: httpx.Retries, elapsed: float) -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=retries) + response = await client.get("https://example.com/connect_timeout") + assert response.status_code == 200 + assert response.elapsed.total_seconds() == elapsed + + +def test_retries_config() -> None: + client = httpx.AsyncClient() + assert client.retries == httpx.Retries() == httpx.Retries(3) + assert client.retries.limits == RetryOnConnectionFailures(3) + assert client.retries.backoff_factor == 0.2 + + client = httpx.AsyncClient(retries=0) + assert client.retries == httpx.Retries(0) + assert client.retries.limits == DontRetry() + + client = httpx.AsyncClient(retries=httpx.Retries(2, backoff_factor=0.1)) + assert client.retries == httpx.Retries(2, backoff_factor=0.1) + assert client.retries.limits == RetryOnConnectionFailures(2) + assert client.retries.backoff_factor == 0.1 + + +# TODO: test custom retry flow that retries on responses. From 919cc9f83ecdcfc418fa8e6554fff453d706b368 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 04:39:43 -0500 Subject: [PATCH 08/22] Disallow retry-on-response (for now) --- httpx/client.py | 9 ++------- httpx/config.py | 4 ++-- httpx/retries.py | 12 ++++++------ 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/httpx/client.py b/httpx/client.py index bec80df974..fadaddcc6e 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -1171,16 +1171,11 @@ async def send_handling_retries( await backend.sleep(delay) else: try: - request = retry_flow.send(response) - except TooManyRetries: - raise + retry_flow.send(None) except StopIteration: return response else: - delay = next(delays) - logger.debug(f"Retrying in {delay} seconds") - await backend.sleep(delay) - continue + raise RuntimeError("Response received, but retry flow didn't stop") async def send_handling_redirects( self, diff --git a/httpx/config.py b/httpx/config.py index 130b143dae..357b9fdfd7 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -6,7 +6,7 @@ import certifi -from .models import URL, Headers, HeaderTypes, Request, Response, URLTypes +from .models import URL, Headers, HeaderTypes, Request, URLTypes from .retries import DontRetry, RetryLimits, RetryOnConnectionFailures from .utils import get_ca_bundle_from_env, get_logger @@ -382,7 +382,7 @@ def get_delays(self) -> typing.Iterator[float]: for n in itertools.count(2): yield self.backoff_factor * (2 ** (n - 2)) - def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + def retry_flow(self, request: Request) -> typing.Generator[Request, None, None]: """ Used by clients to determine what to do when failing to receive a response, or when a response was received. diff --git a/httpx/retries.py b/httpx/retries.py index f2a4532522..ad02784c00 100644 --- a/httpx/retries.py +++ b/httpx/retries.py @@ -7,7 +7,7 @@ PoolTimeout, TooManyRetries, ) -from .models import Request, Response +from .models import Request from .utils import get_logger logger = get_logger(__name__) @@ -18,13 +18,13 @@ class RetryLimits: Base class for retry limiting policies. """ - def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + def retry_flow(self, request: Request) -> typing.Generator[Request, None, None]: """ Execute the retry flow. To dispatch a request, you should `yield` it, and prepare for either: - * The client sending back a response. + * The client managed to send the response. * An `HTTPError` being raised. In each case, decide whether to retry: @@ -40,7 +40,7 @@ class DontRetry(RetryLimits): def __eq__(self, other: typing.Any) -> bool: return type(other) == DontRetry - def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + def retry_flow(self, request: Request) -> typing.Generator[Request, None, None]: # Send the initial request, and never retry. # NOTE: don't raise a `TooManyRetries` exception because this should # really be a no-op implementation. @@ -87,12 +87,12 @@ def _should_retry_for_exception(self, exc: HTTPError) -> bool: return True - def retry_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + def retry_flow(self, request: Request) -> typing.Generator[Request, None, None]: retries_left = self.limit while True: try: - _ = yield request + yield request except HTTPError as exc: # Failed to get a response. From 94d84128b2d2a77f01823e6cd699babb22829e2e Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 04:40:05 -0500 Subject: [PATCH 09/22] Improve coverage --- tests/client/test_retries.py | 74 ++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/tests/client/test_retries.py b/tests/client/test_retries.py index ee00f652d6..e617282bbc 100644 --- a/tests/client/test_retries.py +++ b/tests/client/test_retries.py @@ -10,11 +10,28 @@ from httpx.retries import DontRetry, RetryOnConnectionFailures +def test_retries_config() -> None: + client = httpx.AsyncClient() + assert client.retries == httpx.Retries() == httpx.Retries(3) + assert client.retries.limits == RetryOnConnectionFailures(3) + assert client.retries.backoff_factor == 0.2 + + client = httpx.AsyncClient(retries=0) + assert client.retries == httpx.Retries(0) + assert client.retries.limits == DontRetry() + + client = httpx.AsyncClient(retries=httpx.Retries(2, backoff_factor=0.1)) + assert client.retries == httpx.Retries(2, backoff_factor=0.1) + assert client.retries.limits == RetryOnConnectionFailures(2) + assert client.retries.backoff_factor == 0.1 + + class MockDispatch(AsyncDispatcher): - _ENDPOINTS: typing.Dict[str, typing.Type[httpx.HTTPError]] = { - "/connect_timeout": httpx.ConnectTimeout, - "/pool_timeout": httpx.PoolTimeout, - "/network_error": httpx.NetworkError, + _ENDPOINTS: typing.Dict[str, typing.Tuple[typing.Type[httpx.HTTPError], bool]] = { + "/connect_timeout": (httpx.ConnectTimeout, True), + "/pool_timeout": (httpx.PoolTimeout, True), + "/network_error": (httpx.NetworkError, True), + "/read_timeout": (httpx.ReadTimeout, False), } def __init__(self, succeed_after: int) -> None: @@ -24,9 +41,13 @@ def __init__(self, succeed_after: int) -> None: async def send( self, request: httpx.Request, timeout: TimeoutTypes = None ) -> httpx.Response: - assert request.url.path in self._ENDPOINTS + if request.url.path not in self._ENDPOINTS: + return httpx.Response(httpx.codes.OK, request=request) - exc_cls = self._ENDPOINTS[request.url.path] + exc_cls, is_retryable = self._ENDPOINTS[request.url.path] + + if not is_retryable: + raise exc_cls(request=request) if self.attempts[request.url.path] < self.succeed_after: self.attempts[request.url.path] += 1 @@ -39,6 +60,9 @@ async def send( async def test_no_retries() -> None: client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=0) + response = await client.get("https://example.com") + assert response.status_code == 200 + with pytest.raises(httpx.ConnectTimeout): await client.get("https://example.com/connect_timeout") @@ -53,6 +77,9 @@ async def test_no_retries() -> None: async def test_default_retries() -> None: client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3)) + response = await client.get("https://example.com") + assert response.status_code == 200 + response = await client.get("https://example.com/connect_timeout") assert response.status_code == 200 @@ -67,6 +94,9 @@ async def test_default_retries() -> None: async def test_too_many_retries() -> None: client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=2), retries=1) + response = await client.get("https://example.com") + assert response.status_code == 200 + with pytest.raises(httpx.TooManyRetries): await client.get("https://example.com/connect_timeout") @@ -77,6 +107,19 @@ async def test_too_many_retries() -> None: await client.get("https://example.com/network_error") +@pytest.mark.usefixtures("async_environment") +async def test_exception_not_retryable() -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1), retries=1) + + with pytest.raises(httpx.ReadTimeout): + await client.get("https://example.com/read_timeout") + + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=2), retries=1) + + with pytest.raises(httpx.ReadTimeout): + await client.get("https://example.com/read_timeout") + + @pytest.mark.usefixtures("async_environment") @pytest.mark.parametrize("method", ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]) async def test_retries_idempotent_methods(method: str) -> None: @@ -123,22 +166,3 @@ async def test_retries_backoff(retries: httpx.Retries, elapsed: float) -> None: response = await client.get("https://example.com/connect_timeout") assert response.status_code == 200 assert response.elapsed.total_seconds() == elapsed - - -def test_retries_config() -> None: - client = httpx.AsyncClient() - assert client.retries == httpx.Retries() == httpx.Retries(3) - assert client.retries.limits == RetryOnConnectionFailures(3) - assert client.retries.backoff_factor == 0.2 - - client = httpx.AsyncClient(retries=0) - assert client.retries == httpx.Retries(0) - assert client.retries.limits == DontRetry() - - client = httpx.AsyncClient(retries=httpx.Retries(2, backoff_factor=0.1)) - assert client.retries == httpx.Retries(2, backoff_factor=0.1) - assert client.retries.limits == RetryOnConnectionFailures(2) - assert client.retries.backoff_factor == 0.1 - - -# TODO: test custom retry flow that retries on responses. From 6f9878536a10168a221914c6cef6992a115816d6 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 04:42:11 -0500 Subject: [PATCH 10/22] Don't include initial request in delays iterator --- httpx/client.py | 1 - httpx/config.py | 1 - tests/client/test_retries.py | 10 +++++----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/httpx/client.py b/httpx/client.py index fadaddcc6e..6dfa079cd3 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -1148,7 +1148,6 @@ async def send_handling_retries( retry_flow = retries.retry_flow(request) # Initialize the generators. - next(delays) request = next(retry_flow) while True: diff --git a/httpx/config.py b/httpx/config.py index 357b9fdfd7..1c57a41abc 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -377,7 +377,6 @@ def get_delays(self) -> typing.Iterator[float]: """ Used by clients to determine how long to wait before issuing a new request. """ - yield 0 # Send the initial request. yield 0 # Retry immediately. for n in itertools.count(2): yield self.backoff_factor * (2 ** (n - 2)) diff --git a/tests/client/test_retries.py b/tests/client/test_retries.py index e617282bbc..ac9be46189 100644 --- a/tests/client/test_retries.py +++ b/tests/client/test_retries.py @@ -142,14 +142,14 @@ async def test_no_retries_non_idempotent_methods() -> None: @pytest.mark.parametrize( "retries, delays", [ - (httpx.Retries(), [0, 0, 0.2, 0.4, 0.8, 1.6]), - (httpx.Retries(backoff_factor=0.1), [0, 0, 0.1, 0.2, 0.4, 0.8]), + (httpx.Retries(), [0, 0.2, 0.4, 0.8, 1.6]), + (httpx.Retries(backoff_factor=0.1), [0, 0.1, 0.2, 0.4, 0.8]), ], ) def test_retries_delays_sequence( retries: httpx.Retries, delays: typing.List[int] ) -> None: - sample_delays = list(itertools.islice(retries.get_delays(), 6)) + sample_delays = list(itertools.islice(retries.get_delays(), 5)) assert sample_delays == delays @@ -157,8 +157,8 @@ def test_retries_delays_sequence( @pytest.mark.parametrize( "retries, elapsed", [ - (httpx.Retries(), pytest.approx(0 + 0 + 0.2 + 0.4, rel=0.1)), - (httpx.Retries(backoff_factor=0.1), pytest.approx(0 + 0 + 0.1 + 0.2, rel=0.2)), + (httpx.Retries(), pytest.approx(0 + 0.2 + 0.4, rel=0.1)), + (httpx.Retries(backoff_factor=0.1), pytest.approx(0 + 0.1 + 0.2, rel=0.2)), ], ) async def test_retries_backoff(retries: httpx.Retries, elapsed: float) -> None: From 00ba0587e493b9b2cb48345491818dd6e8637423 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 05:03:10 -0500 Subject: [PATCH 11/22] Update docs --- docs/advanced.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index 6154bb6ece..2a6d72b9d7 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -469,9 +469,24 @@ Response <200 OK> ## Retries -Communicating with a peer over a network is by essence subject to errors. HTTPX provides built-in retry functionality to increase the resilience to unexpected issues such as network faults or connection issues. +Communicating with a peer over a network is by essence subject to errors. HTTPX provides built-in retry functionality to increase the resilience to unexpected issues. -The default behavior is to retry at most 3 times on connection and network errors before marking the request as failed and bubbling up any exceptions. The delay between retries is increased each time to prevent overloading the requested server. +By default, HTTPX will retry **at most 3 times** on connection failures. This means: + +* Failures to establish or acquire a connection (`ConnectTimeout`, `PoolTimeout`). +* Failures to keep the connection open (`NetworkError`). + +If a response is not obtained after these attempts, any exception is bubbled up. + +The delay between each retry is increased exponentially to prevent overloading the requested host. + +!!! important + HTTPX will **NOT** retry on failures that aren't related to establishing or maintaining connections. + + In particular, this includes: + + * Errors related to data transfer, such as `ReadTimeout` or `ProtocolError`. + * HTTP error responses (4xx, 5xx), such as `429 Too Many Requests` or `503 Service Unavailable`. ### Setting and disabling retries From 0d436e30bd2b5779bcc79a9489ad11a87a4e1d1d Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 05:40:37 -0500 Subject: [PATCH 12/22] Add sync retries --- httpx/backends/sync.py | 6 +++++ httpx/client.py | 55 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 httpx/backends/sync.py diff --git a/httpx/backends/sync.py b/httpx/backends/sync.py new file mode 100644 index 0000000000..c2804e3f62 --- /dev/null +++ b/httpx/backends/sync.py @@ -0,0 +1,6 @@ +import time + + +class SyncBackend: + def sleep(self, seconds: float) -> None: + time.sleep(seconds) diff --git a/httpx/client.py b/httpx/client.py index 6dfa079cd3..e35192a33a 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -6,6 +6,7 @@ from .auth import Auth, AuthTypes, BasicAuth, FunctionAuth from .backends.base import ConcurrencyBackend, lookup_backend +from .backends.sync import SyncBackend from .config import ( DEFAULT_MAX_REDIRECTS, DEFAULT_POOL_LIMITS, @@ -484,6 +485,7 @@ def __init__( ) for key, proxy in proxy_map.items() } + self.backend = SyncBackend() def init_dispatch( self, @@ -588,10 +590,16 @@ def send( timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) + retries = self.retries + auth = self.build_auth(request, auth) - response = self.send_handling_redirects( - request, auth=auth, timeout=timeout, allow_redirects=allow_redirects, + response = self.send_handling_retries( + request, + auth=auth, + retries=retries, + timeout=timeout, + allow_redirects=allow_redirects, ) if not stream: @@ -602,6 +610,47 @@ def send( return response + def send_handling_retries( + self, + request: Request, + auth: Auth, + retries: Retries, + timeout: Timeout, + allow_redirects: bool = True, + ) -> Response: + backend = self.backend + delays = retries.get_delays() + retry_flow = retries.retry_flow(request) + + # Initialize the generators. + request = next(retry_flow) + + while True: + try: + response = self.send_handling_redirects( + request, + auth=auth, + timeout=timeout, + allow_redirects=allow_redirects, + ) + except HTTPError as exc: + logger.debug(f"HTTP Request failed: {exc!r}") + try: + request = retry_flow.throw(type(exc), exc, exc.__traceback__) + except (TooManyRetries, HTTPError): + raise + else: + delay = next(delays) + logger.debug(f"Retrying in {delay} seconds") + backend.sleep(delay) + else: + try: + retry_flow.send(None) + except StopIteration: + return response + else: + raise RuntimeError("Response received, but retry flow didn't stop") + def send_handling_redirects( self, request: Request, @@ -1121,8 +1170,8 @@ async def send( response = await self.send_handling_retries( request, auth=auth, - timeout=timeout, retries=retries, + timeout=timeout, allow_redirects=allow_redirects, ) From 76a5b9cb15b44a142ad57005ca211a1c74a928a9 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 14:00:54 -0500 Subject: [PATCH 13/22] Drop RetryLimits internal interface, rearrange tests --- httpx/__init__.py | 2 - httpx/client.py | 70 ++++++++++------------ httpx/config.py | 51 +++++++++++----- httpx/retries.py | 112 ----------------------------------- tests/client/test_retries.py | 78 ++++++++++++------------ 5 files changed, 104 insertions(+), 209 deletions(-) delete mode 100644 httpx/retries.py diff --git a/httpx/__init__.py b/httpx/__init__.py index 6864a40e7c..092adf315c 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -30,7 +30,6 @@ WriteTimeout, ) from .models import URL, Cookies, Headers, QueryParams, Request, Response -from .retries import RetryLimits from .status_codes import StatusCode, codes __all__ = [ @@ -58,7 +57,6 @@ "Proxy", "Timeout", "Retries", - "RetryLimits", "TooManyRetries", "ConnectTimeout", "CookieConflict", diff --git a/httpx/client.py b/httpx/client.py index e35192a33a..e5ac4845cf 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -619,37 +619,32 @@ def send_handling_retries( allow_redirects: bool = True, ) -> Response: backend = self.backend + retries_left = retries.limit delays = retries.get_delays() - retry_flow = retries.retry_flow(request) - - # Initialize the generators. - request = next(retry_flow) while True: try: - response = self.send_handling_redirects( + return self.send_handling_redirects( request, auth=auth, timeout=timeout, allow_redirects=allow_redirects, ) except HTTPError as exc: - logger.debug(f"HTTP Request failed: {exc!r}") - try: - request = retry_flow.throw(type(exc), exc, exc.__traceback__) - except (TooManyRetries, HTTPError): + if not retries.limit or not retries.should_retry_on_exception(exc): + # We shouldn't retry at all in these cases, so let's re-raise + # immediately to avoid polluting logs or the exception stack. raise - else: - delay = next(delays) - logger.debug(f"Retrying in {delay} seconds") - backend.sleep(delay) - else: - try: - retry_flow.send(None) - except StopIteration: - return response - else: - raise RuntimeError("Response received, but retry flow didn't stop") + + logger.debug(f"HTTP Request failed: {exc!r}") + + if not retries_left: + raise TooManyRetries(exc, request=request) + + retries_left -= 1 + delay = next(delays) + logger.debug(f"Retrying in {delay} seconds") + backend.sleep(delay) def send_handling_redirects( self, @@ -1193,37 +1188,32 @@ async def send_handling_retries( ) -> Response: backend = lookup_backend() + retries_left = retries.limit delays = retries.get_delays() - retry_flow = retries.retry_flow(request) - - # Initialize the generators. - request = next(retry_flow) while True: try: - response = await self.send_handling_redirects( + return await self.send_handling_redirects( request, auth=auth, timeout=timeout, allow_redirects=allow_redirects, ) except HTTPError as exc: - logger.debug(f"HTTP Request failed: {exc!r}") - try: - request = retry_flow.throw(type(exc), exc, exc.__traceback__) - except (TooManyRetries, HTTPError): + if not retries.limit or not retries.should_retry_on_exception(exc): + # We shouldn't retry at all in these cases, so let's re-raise + # immediately to avoid polluting logs or the exception stack. raise - else: - delay = next(delays) - logger.debug(f"Retrying in {delay} seconds") - await backend.sleep(delay) - else: - try: - retry_flow.send(None) - except StopIteration: - return response - else: - raise RuntimeError("Response received, but retry flow didn't stop") + + logger.debug(f"HTTP Request failed: {exc!r}") + + if not retries_left: + raise TooManyRetries(exc, request=request) + + retries_left -= 1 + delay = next(delays) + logger.debug(f"Retrying in {delay} seconds") + await backend.sleep(delay) async def send_handling_redirects( self, diff --git a/httpx/config.py b/httpx/config.py index 1c57a41abc..e5db64afac 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -6,8 +6,8 @@ import certifi -from .models import URL, Headers, HeaderTypes, Request, URLTypes -from .retries import DontRetry, RetryLimits, RetryOnConnectionFailures +from .exceptions import ConnectTimeout, HTTPError, NetworkError, PoolTimeout +from .models import URL, Headers, HeaderTypes, URLTypes from .utils import get_ca_bundle_from_env, get_logger CertTypes = typing.Union[str, typing.Tuple[str, str], typing.Tuple[str, str, str]] @@ -344,35 +344,61 @@ class Retries: """ Retries configuration. - Defines the retry limiting policy, and implements a configurable - exponential backoff algorithm. + Defines the maximum amount of connection failures to retry on, and + implements a configurable exponential backoff algorithm. """ + _RETRYABLE_EXCEPTIONS: typing.Sequence[typing.Type[HTTPError]] = ( + ConnectTimeout, + PoolTimeout, + NetworkError, + ) + _RETRYABLE_METHODS: typing.Container[str] = frozenset( + ("HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE") + ) + def __init__( self, retries: RetriesTypes = 3, *, backoff_factor: float = None ) -> None: if isinstance(retries, int): - limits = RetryOnConnectionFailures(retries) if retries > 0 else DontRetry() - elif isinstance(retries, Retries): + limit = retries + else: + assert isinstance(retries, Retries) assert backoff_factor is None backoff_factor = retries.backoff_factor - limits = retries.limits + limit = retries.limit if backoff_factor is None: backoff_factor = 0.2 + assert limit >= 0 assert backoff_factor > 0 - self.limits: RetryLimits = limits + self.limit: int = limit self.backoff_factor: float = backoff_factor def __eq__(self, other: typing.Any) -> bool: return ( isinstance(other, Retries) - and self.limits == other.limits + and self.limit == other.limit and self.backoff_factor == other.backoff_factor ) + @classmethod + def should_retry_on_exception(cls, exc: HTTPError) -> bool: + for exc_cls in cls._RETRYABLE_EXCEPTIONS: + if isinstance(exc, exc_cls): + break + else: + return False + + assert exc.request is not None + method = exc.request.method.upper() + if method not in cls._RETRYABLE_METHODS: + return False + + return True + def get_delays(self) -> typing.Iterator[float]: """ Used by clients to determine how long to wait before issuing a new request. @@ -381,13 +407,6 @@ def get_delays(self) -> typing.Iterator[float]: for n in itertools.count(2): yield self.backoff_factor * (2 ** (n - 2)) - def retry_flow(self, request: Request) -> typing.Generator[Request, None, None]: - """ - Used by clients to determine what to do when failing to receive a response, - or when a response was received. - """ - yield from self.limits.retry_flow(request) - DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0) DEFAULT_RETRIES_CONFIG = Retries(3, backoff_factor=0.2) diff --git a/httpx/retries.py b/httpx/retries.py deleted file mode 100644 index ad02784c00..0000000000 --- a/httpx/retries.py +++ /dev/null @@ -1,112 +0,0 @@ -import typing - -from .exceptions import ( - ConnectTimeout, - HTTPError, - NetworkError, - PoolTimeout, - TooManyRetries, -) -from .models import Request -from .utils import get_logger - -logger = get_logger(__name__) - - -class RetryLimits: - """ - Base class for retry limiting policies. - """ - - def retry_flow(self, request: Request) -> typing.Generator[Request, None, None]: - """ - Execute the retry flow. - - To dispatch a request, you should `yield` it, and prepare for either: - - * The client managed to send the response. - * An `HTTPError` being raised. - - In each case, decide whether to retry: - - * If so, continue yielding, unless a maximum number of retries was exceeded. - In that case, raise a `TooManyRetries` exception. - * Otherwise, `return`, or `raise` the exception. - """ - raise NotImplementedError # pragma: no cover - - -class DontRetry(RetryLimits): - def __eq__(self, other: typing.Any) -> bool: - return type(other) == DontRetry - - def retry_flow(self, request: Request) -> typing.Generator[Request, None, None]: - # Send the initial request, and never retry. - # NOTE: don't raise a `TooManyRetries` exception because this should - # really be a no-op implementation. - yield request - - -class RetryOnConnectionFailures(RetryLimits): - """ - Retry when failing to establish a connection, or when a network - error occurred. - """ - - _RETRYABLE_EXCEPTIONS: typing.Sequence[typing.Type[HTTPError]] = ( - ConnectTimeout, - PoolTimeout, - NetworkError, - ) - _RETRYABLE_METHODS: typing.Container[str] = frozenset( - ("HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE") - ) - - def __init__(self, limit: int = 3) -> None: - assert limit >= 0 - self.limit = limit - - def __eq__(self, other: typing.Any) -> bool: - return ( - isinstance(other, RetryOnConnectionFailures) and self.limit == other.limit - ) - - def _should_retry_for_exception(self, exc: HTTPError) -> bool: - for exc_cls in self._RETRYABLE_EXCEPTIONS: - if isinstance(exc, exc_cls): - break - else: - logger.debug(f"not_retryable exc_type={type(exc)}") - return False - - assert exc.request is not None - method = exc.request.method.upper() - if method not in self._RETRYABLE_METHODS: - logger.debug(f"not_retryable method={method!r}") - return False - - return True - - def retry_flow(self, request: Request) -> typing.Generator[Request, None, None]: - retries_left = self.limit - - while True: - try: - yield request - except HTTPError as exc: - # Failed to get a response. - - if not retries_left: - raise TooManyRetries(exc, request=request) - - if self._should_retry_for_exception(exc): - retries_left -= 1 - continue - - # Raise the exception for other retry limits involved to handle, - # or for bubbling up to the client. - raise - else: - # We managed to get a response without connection/network - # failures, so we're done here. - return diff --git a/tests/client/test_retries.py b/tests/client/test_retries.py index ac9be46189..0ccf8676c6 100644 --- a/tests/client/test_retries.py +++ b/tests/client/test_retries.py @@ -7,25 +7,52 @@ import httpx from httpx.config import TimeoutTypes from httpx.dispatch.base import AsyncDispatcher -from httpx.retries import DontRetry, RetryOnConnectionFailures def test_retries_config() -> None: client = httpx.AsyncClient() assert client.retries == httpx.Retries() == httpx.Retries(3) - assert client.retries.limits == RetryOnConnectionFailures(3) + assert client.retries.limit == 3 assert client.retries.backoff_factor == 0.2 client = httpx.AsyncClient(retries=0) - assert client.retries == httpx.Retries(0) - assert client.retries.limits == DontRetry() + assert client.retries.limit == 0 client = httpx.AsyncClient(retries=httpx.Retries(2, backoff_factor=0.1)) assert client.retries == httpx.Retries(2, backoff_factor=0.1) - assert client.retries.limits == RetryOnConnectionFailures(2) + assert client.retries.limit == 2 assert client.retries.backoff_factor == 0.1 +@pytest.mark.parametrize( + "retries, delays", + [ + (httpx.Retries(), [0, 0.2, 0.4, 0.8, 1.6]), + (httpx.Retries(backoff_factor=0.1), [0, 0.1, 0.2, 0.4, 0.8]), + ], +) +def test_retries_delays_sequence( + retries: httpx.Retries, delays: typing.List[int] +) -> None: + sample_delays = list(itertools.islice(retries.get_delays(), 5)) + assert sample_delays == delays + + +@pytest.mark.usefixtures("async_environment") +@pytest.mark.parametrize( + "retries, elapsed", + [ + (httpx.Retries(), pytest.approx(0 + 0.2 + 0.4, rel=0.1)), + (httpx.Retries(backoff_factor=0.1), pytest.approx(0 + 0.1 + 0.2, rel=0.2)), + ], +) +async def test_retries_backoff(retries: httpx.Retries, elapsed: float) -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=retries) + response = await client.get("https://example.com/connect_timeout") + assert response.status_code == 200 + assert response.elapsed.total_seconds() == elapsed + + class MockDispatch(AsyncDispatcher): _ENDPOINTS: typing.Dict[str, typing.Tuple[typing.Type[httpx.HTTPError], bool]] = { "/connect_timeout": (httpx.ConnectTimeout, True), @@ -58,7 +85,7 @@ async def send( @pytest.mark.usefixtures("async_environment") async def test_no_retries() -> None: - client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=0) + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1), retries=0) response = await client.get("https://example.com") assert response.status_code == 200 @@ -109,13 +136,15 @@ async def test_too_many_retries() -> None: @pytest.mark.usefixtures("async_environment") async def test_exception_not_retryable() -> None: + # If ReadTimeout was retryable, the call would succeed after one retry, but + # it shouldn't. client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1), retries=1) - with pytest.raises(httpx.ReadTimeout): await client.get("https://example.com/read_timeout") + # If ReadTimeout was retryable, this call would fail with 'TooManyRetries', + # but it shouldn't. client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=2), retries=1) - with pytest.raises(httpx.ReadTimeout): await client.get("https://example.com/read_timeout") @@ -129,40 +158,11 @@ async def test_retries_idempotent_methods(method: str) -> None: @pytest.mark.usefixtures("async_environment") -async def test_no_retries_non_idempotent_methods() -> None: +async def test_non_idempotent_methods_not_retryable() -> None: client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1)) + # These would succeed if POST or PATCH triggered retries, but they shouldn't. with pytest.raises(httpx.ConnectTimeout): await client.post("https://example.com/connect_timeout") - with pytest.raises(httpx.PoolTimeout): await client.patch("https://example.com/pool_timeout") - - -@pytest.mark.parametrize( - "retries, delays", - [ - (httpx.Retries(), [0, 0.2, 0.4, 0.8, 1.6]), - (httpx.Retries(backoff_factor=0.1), [0, 0.1, 0.2, 0.4, 0.8]), - ], -) -def test_retries_delays_sequence( - retries: httpx.Retries, delays: typing.List[int] -) -> None: - sample_delays = list(itertools.islice(retries.get_delays(), 5)) - assert sample_delays == delays - - -@pytest.mark.usefixtures("async_environment") -@pytest.mark.parametrize( - "retries, elapsed", - [ - (httpx.Retries(), pytest.approx(0 + 0.2 + 0.4, rel=0.1)), - (httpx.Retries(backoff_factor=0.1), pytest.approx(0 + 0.1 + 0.2, rel=0.2)), - ], -) -async def test_retries_backoff(retries: httpx.Retries, elapsed: float) -> None: - client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=retries) - response = await client.get("https://example.com/connect_timeout") - assert response.status_code == 200 - assert response.elapsed.total_seconds() == elapsed From e9c154206ed1c4319cf60289f386576c7c4f809f Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 14:37:17 -0500 Subject: [PATCH 14/22] Switch to off-by-default retries --- httpx/config.py | 4 +-- tests/client/test_retries.py | 48 ++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/httpx/config.py b/httpx/config.py index ae30767c8d..401a5dfded 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -373,7 +373,7 @@ class Retries: ) def __init__( - self, retries: RetriesTypes = 3, *, backoff_factor: float = None + self, retries: RetriesTypes = 0, *, backoff_factor: float = None ) -> None: if isinstance(retries, int): limit = retries @@ -424,6 +424,6 @@ def get_delays(self) -> typing.Iterator[float]: DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0) -DEFAULT_RETRIES_CONFIG = Retries(3, backoff_factor=0.2) +DEFAULT_RETRIES_CONFIG = Retries(0, backoff_factor=0.2) DEFAULT_POOL_LIMITS = PoolLimits(soft_limit=10, hard_limit=100) DEFAULT_MAX_REDIRECTS = 20 diff --git a/tests/client/test_retries.py b/tests/client/test_retries.py index 0ccf8676c6..2b6809e09e 100644 --- a/tests/client/test_retries.py +++ b/tests/client/test_retries.py @@ -11,13 +11,13 @@ def test_retries_config() -> None: client = httpx.AsyncClient() - assert client.retries == httpx.Retries() == httpx.Retries(3) + assert client.retries == httpx.Retries() == httpx.Retries(0) + assert client.retries.limit == 0 + + client = httpx.AsyncClient(retries=3) assert client.retries.limit == 3 assert client.retries.backoff_factor == 0.2 - client = httpx.AsyncClient(retries=0) - assert client.retries.limit == 0 - client = httpx.AsyncClient(retries=httpx.Retries(2, backoff_factor=0.1)) assert client.retries == httpx.Retries(2, backoff_factor=0.1) assert client.retries.limit == 2 @@ -38,21 +38,6 @@ def test_retries_delays_sequence( assert sample_delays == delays -@pytest.mark.usefixtures("async_environment") -@pytest.mark.parametrize( - "retries, elapsed", - [ - (httpx.Retries(), pytest.approx(0 + 0.2 + 0.4, rel=0.1)), - (httpx.Retries(backoff_factor=0.1), pytest.approx(0 + 0.1 + 0.2, rel=0.2)), - ], -) -async def test_retries_backoff(retries: httpx.Retries, elapsed: float) -> None: - client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=retries) - response = await client.get("https://example.com/connect_timeout") - assert response.status_code == 200 - assert response.elapsed.total_seconds() == elapsed - - class MockDispatch(AsyncDispatcher): _ENDPOINTS: typing.Dict[str, typing.Tuple[typing.Type[httpx.HTTPError], bool]] = { "/connect_timeout": (httpx.ConnectTimeout, True), @@ -85,7 +70,7 @@ async def send( @pytest.mark.usefixtures("async_environment") async def test_no_retries() -> None: - client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1), retries=0) + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1)) response = await client.get("https://example.com") assert response.status_code == 200 @@ -101,8 +86,8 @@ async def test_no_retries() -> None: @pytest.mark.usefixtures("async_environment") -async def test_default_retries() -> None: - client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3)) +async def test_retries() -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=3) response = await client.get("https://example.com") assert response.status_code == 200 @@ -152,17 +137,32 @@ async def test_exception_not_retryable() -> None: @pytest.mark.usefixtures("async_environment") @pytest.mark.parametrize("method", ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]) async def test_retries_idempotent_methods(method: str) -> None: - client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1)) + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1), retries=3) response = await client.request(method, "https://example.com/connect_timeout") assert response.status_code == 200 @pytest.mark.usefixtures("async_environment") async def test_non_idempotent_methods_not_retryable() -> None: - client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1)) + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=1), retries=3) # These would succeed if POST or PATCH triggered retries, but they shouldn't. with pytest.raises(httpx.ConnectTimeout): await client.post("https://example.com/connect_timeout") with pytest.raises(httpx.PoolTimeout): await client.patch("https://example.com/pool_timeout") + + +@pytest.mark.usefixtures("async_environment") +@pytest.mark.parametrize( + "retries, elapsed", + [ + (3, pytest.approx(0 + 0.2 + 0.4, rel=0.1)), + (httpx.Retries(3, backoff_factor=0.1), pytest.approx(0 + 0.1 + 0.2, rel=0.2)), + ], +) +async def test_retries_backoff(retries: httpx.Retries, elapsed: float) -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=retries) + response = await client.get("https://example.com/connect_timeout") + assert response.status_code == 200 + assert response.elapsed.total_seconds() == elapsed From cd4ded167d32486a5ab3241ea6c9ae25d83f315a Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 14:37:23 -0500 Subject: [PATCH 15/22] Update docs --- docs/advanced.md | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index 0db33551f6..3c52404356 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -474,50 +474,47 @@ Response <200 OK> ## Retries -Communicating with a peer over a network is by essence subject to errors. HTTPX provides built-in retry functionality to increase the resilience to unexpected issues. +Communicating with a peer over a network is by essence subject to errors. HTTPX provides built-in retry functionality to increase the resilience to connection issues. -By default, HTTPX will retry **at most 3 times** on connection failures. This means: +### Enabling retries -* Failures to establish or acquire a connection (`ConnectTimeout`, `PoolTimeout`). -* Failures to keep the connection open (`NetworkError`). +Retries are disabled by default. You can enable them on a client instance using the `retries` parameter: -If a response is not obtained after these attempts, any exception is bubbled up. +```python +# Retry at most 3 times on connection failures. +client = httpx.Client(retries=3) +``` -The delay between each retry is increased exponentially to prevent overloading the requested host. +When retries are enabled, HTTPX will retry sending the request up to the specified number of times. This behavior is restricted to **connection failures only**, i.e.: -!!! important - HTTPX will **NOT** retry on failures that aren't related to establishing or maintaining connections. +* Failures to establish or acquire a connection (`ConnectTimeout`, `PoolTimeout`). +* Failures to keep the connection open (`NetworkError`). - In particular, this includes: +!!! important + HTTPX will **NOT** retry on failures that aren't related to establishing or maintaining connections. This includes in particular: * Errors related to data transfer, such as `ReadTimeout` or `ProtocolError`. * HTTP error responses (4xx, 5xx), such as `429 Too Many Requests` or `503 Service Unavailable`. -### Setting and disabling retries - -You can set the retry behavior on a client instance, which results in the given behavior being used for all requests made with this client: +If HTTPX could not get a response after the specified number of retries, a `TooManyRetries` exception is raised. -```python -client = httpx.Client() # Retry at most 3 times on connection failures. -client = httpx.Client(retries=5) # Retry at most 5 times on connection failures. -client = httpx.Client(retries=0) # Disable retries. -``` +The delay between each retry is increased exponentially to prevent overloading the requested host. ### Fine-tuning the retries configuration When instantiating a client, the `retries` argument may be one of the following... -* An integer, representing the maximum number connection failures to retry on. Use `0` to disable retries entirely. +* An integer, representing the maximum number of connection failures to retry on. The default is `0`. ```python client = httpx.Client(retries=5) ``` -* An `httpx.Retries()` instance. It accepts the number of connection failures to retry on as a positional argument. The `backoff_factor` keyword argument that specifies how fast the time to wait before issuing a retry request should be increased. By default this is `0.2`, which corresponds to issuing a new request after `(0s, 0.2s, 0.4s, 0.8s, ...)`. (Note that a lot of errors are immediately resolved by retrying, so HTTPX will always issue the initial retry right away.) +* An `httpx.Retries()` instance. This can be used to customize the `backoff_factor`, which defines the increase rate of the time to wait between retries. By default it is `0.2`, which corresponds to issuing a new request after `(0s, 0.2s, 0.4s, 0.8s, ...)`. (Note that most connection failures are immediately resolved by retrying, so HTTPX will always issue the initial retry right away.) ```python # Retry at most 5 times on connection failures, -# and issue new requests after `(0s, 0.5s, 1s, 2s, 4s, ...)` +# and issue new requests after `(0s, 0.5s, 1s, 2s, 4s, ...)`. retries = httpx.Retries(5, backoff_factor=0.5) client = httpx.Client(retries=retries) ``` From 6570b9f1f7768533e690b05f97cb546de2fcb2ea Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 14:41:41 -0500 Subject: [PATCH 16/22] Add 'retries' to sync client --- httpx/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/httpx/client.py b/httpx/client.py index 70354daea8..fd456f3b4a 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -427,6 +427,8 @@ class Client(BaseClient): URLs. * **timeout** - *(optional)* The timeout configuration to use when sending requests. + * **retries** - *(optional)* The maximum number of connection failures to + retry on. * **pool_limits** - *(optional)* The connection pool configuration to use when determining the maximum number of concurrently open HTTP connections. * **max_redirects** - *(optional)* The maximum number of redirect responses @@ -452,6 +454,7 @@ def __init__( cert: CertTypes = None, proxies: ProxiesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, pool_limits: PoolLimits = DEFAULT_POOL_LIMITS, max_redirects: int = DEFAULT_MAX_REDIRECTS, base_url: URLTypes = None, @@ -465,6 +468,7 @@ def __init__( headers=headers, cookies=cookies, timeout=timeout, + retries=retries, max_redirects=max_redirects, base_url=base_url, trust_env=trust_env, From 19a900712ae331f62863259381260d4b166b6d12 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 15:06:17 -0500 Subject: [PATCH 17/22] Tweak stacktrace in sync case --- httpx/dispatch/urllib3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/httpx/dispatch/urllib3.py b/httpx/dispatch/urllib3.py index 2728170c14..9999980c9a 100644 --- a/httpx/dispatch/urllib3.py +++ b/httpx/dispatch/urllib3.py @@ -4,7 +4,7 @@ import typing import urllib3 -from urllib3.exceptions import MaxRetryError, SSLError +from urllib3.exceptions import SSLError, NewConnectionError from ..config import ( DEFAULT_POOL_LIMITS, @@ -94,7 +94,7 @@ def send(self, request: Request, timeout: Timeout = None) -> Response: content_length = int(request.headers.get("Content-Length", "0")) body = request.stream if chunked or content_length else None - with as_network_error(MaxRetryError, SSLError, socket.error): + with as_network_error(NewConnectionError, SSLError, socket.error): conn = self.pool.urlopen( method=request.method, url=str(request.url), @@ -102,7 +102,7 @@ def send(self, request: Request, timeout: Timeout = None) -> Response: body=body, redirect=False, assert_same_host=False, - retries=0, + retries=False, preload_content=False, chunked=chunked, timeout=urllib3_timeout, From 5db9eb9b1da645675fecd3bee742f6ffd6f60e27 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 15:55:01 -0500 Subject: [PATCH 18/22] Lint --- httpx/dispatch/urllib3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpx/dispatch/urllib3.py b/httpx/dispatch/urllib3.py index 9999980c9a..d37f9243ff 100644 --- a/httpx/dispatch/urllib3.py +++ b/httpx/dispatch/urllib3.py @@ -4,7 +4,7 @@ import typing import urllib3 -from urllib3.exceptions import SSLError, NewConnectionError +from urllib3.exceptions import NewConnectionError, SSLError from ..config import ( DEFAULT_POOL_LIMITS, From c9abd751ed1916d9cca386e16298e4db4b9849e4 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 20 Jan 2020 15:55:26 -0500 Subject: [PATCH 19/22] Add per-request retries --- docs/advanced.md | 36 ++++++++++++---------- httpx/api.py | 31 +++++++++++++++++-- httpx/client.py | 58 ++++++++++++++++++++++++++++++++---- httpx/config.py | 4 +-- tests/client/test_retries.py | 6 ++-- tests/test_timeouts.py | 6 ++-- 6 files changed, 107 insertions(+), 34 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index 3c52404356..de5a9af3f3 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -476,16 +476,7 @@ Response <200 OK> Communicating with a peer over a network is by essence subject to errors. HTTPX provides built-in retry functionality to increase the resilience to connection issues. -### Enabling retries - -Retries are disabled by default. You can enable them on a client instance using the `retries` parameter: - -```python -# Retry at most 3 times on connection failures. -client = httpx.Client(retries=3) -``` - -When retries are enabled, HTTPX will retry sending the request up to the specified number of times. This behavior is restricted to **connection failures only**, i.e.: +Retries are disabled by default. When retries are enabled, HTTPX will retry sending the request up to the specified number of times. This behavior is restricted to **connection failures only**, i.e.: * Failures to establish or acquire a connection (`ConnectTimeout`, `PoolTimeout`). * Failures to keep the connection open (`NetworkError`). @@ -500,17 +491,32 @@ If HTTPX could not get a response after the specified number of retries, a `TooM The delay between each retry is increased exponentially to prevent overloading the requested host. -### Fine-tuning the retries configuration +### Enabling retries + +You can enable retries for a given request: -When instantiating a client, the `retries` argument may be one of the following... +```python +# Using the top-level API: +response = httpx.get("https://example.org", retries=3) -* An integer, representing the maximum number of connection failures to retry on. The default is `0`. +# Using a client instance: +with httpx.Client() as client: + response = client.get("https://example.org", retries=3) +``` + +Or enable them on a client instance, which results in the given `retries` being used as a default for requests made with this client: ```python -client = httpx.Client(retries=5) +# Retry at most 3 times on connection failures everywhere. +httpx.Client(retries=3) ``` -* An `httpx.Retries()` instance. This can be used to customize the `backoff_factor`, which defines the increase rate of the time to wait between retries. By default it is `0.2`, which corresponds to issuing a new request after `(0s, 0.2s, 0.4s, 0.8s, ...)`. (Note that most connection failures are immediately resolved by retrying, so HTTPX will always issue the initial retry right away.) +### Fine-tuning the retries configuration + +When enabling retries, the `retries` argument can also be an `httpx.Retries()` instance. It accepts the following arguments: + +* An integer, given as a required positional argument, representing the maximum number of connection failures to retry on. +* `backoff_factor` (optional), which defines the increase rate of the time to wait between retries. By default this is `0.2`, which corresponds to issuing a new request after `(0s, 0.2s, 0.4s, 0.8s, ...)`. (Note that most connection failures are immediately resolved by retrying, so HTTPX will always issue the initial retry right away.) ```python # Retry at most 5 times on connection failures, diff --git a/httpx/api.py b/httpx/api.py index 7fbbd30811..74348575ba 100644 --- a/httpx/api.py +++ b/httpx/api.py @@ -2,7 +2,14 @@ from .auth import AuthTypes from .client import Client, StreamContextManager -from .config import DEFAULT_TIMEOUT_CONFIG, CertTypes, TimeoutTypes, VerifyTypes +from .config import ( + DEFAULT_RETRIES_CONFIG, + DEFAULT_TIMEOUT_CONFIG, + CertTypes, + RetriesTypes, + TimeoutTypes, + VerifyTypes, +) from .models import ( CookieTypes, HeaderTypes, @@ -26,6 +33,7 @@ def request( headers: HeaderTypes = None, cookies: CookieTypes = None, auth: AuthTypes = None, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, allow_redirects: bool = True, verify: VerifyTypes = True, @@ -54,6 +62,8 @@ def request( request. * **auth** - *(optional)* An authentication class to use when sending the request. + * **retries** - *(optional)* The maximum number of connection failures to + retry on. * **timeout** - *(optional)* The timeout configuration to use when sending the request. * **allow_redirects** - *(optional)* Enables or disables HTTP redirects. @@ -81,7 +91,7 @@ def request( ``` """ with Client( - cert=cert, verify=verify, timeout=timeout, trust_env=trust_env, + cert=cert, verify=verify, retries=retries, timeout=timeout, trust_env=trust_env, ) as client: return client.request( method=method, @@ -108,13 +118,14 @@ def stream( headers: HeaderTypes = None, cookies: CookieTypes = None, auth: AuthTypes = None, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, allow_redirects: bool = True, verify: VerifyTypes = True, cert: CertTypes = None, trust_env: bool = True, ) -> StreamContextManager: - client = Client(cert=cert, verify=verify, trust_env=trust_env) + client = Client(cert=cert, verify=verify, retries=retries, trust_env=trust_env) request = Request( method=method, url=url, @@ -145,6 +156,7 @@ def get( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -166,6 +178,7 @@ def get( allow_redirects=allow_redirects, cert=cert, verify=verify, + retries=retries, timeout=timeout, trust_env=trust_env, ) @@ -181,6 +194,7 @@ def options( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -202,6 +216,7 @@ def options( allow_redirects=allow_redirects, cert=cert, verify=verify, + retries=retries, timeout=timeout, trust_env=trust_env, ) @@ -217,6 +232,7 @@ def head( allow_redirects: bool = False, # Note: Differs to usual default. cert: CertTypes = None, verify: VerifyTypes = True, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -240,6 +256,7 @@ def head( allow_redirects=allow_redirects, cert=cert, verify=verify, + retries=retries, timeout=timeout, trust_env=trust_env, ) @@ -258,6 +275,7 @@ def post( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -279,6 +297,7 @@ def post( allow_redirects=allow_redirects, cert=cert, verify=verify, + retries=retries, timeout=timeout, trust_env=trust_env, ) @@ -297,6 +316,7 @@ def put( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -318,6 +338,7 @@ def put( allow_redirects=allow_redirects, cert=cert, verify=verify, + retries=retries, timeout=timeout, trust_env=trust_env, ) @@ -336,6 +357,7 @@ def patch( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -357,6 +379,7 @@ def patch( allow_redirects=allow_redirects, cert=cert, verify=verify, + retries=retries, timeout=timeout, trust_env=trust_env, ) @@ -372,6 +395,7 @@ def delete( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, + retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -393,6 +417,7 @@ def delete( allow_redirects=allow_redirects, cert=cert, verify=verify, + retries=retries, timeout=timeout, trust_env=trust_env, ) diff --git a/httpx/client.py b/httpx/client.py index fd456f3b4a..63686c7dd6 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -425,10 +425,10 @@ class Client(BaseClient): file, key file, password). * **proxies** - *(optional)* A dictionary mapping HTTP protocols to proxy URLs. - * **timeout** - *(optional)* The timeout configuration to use when sending - requests. * **retries** - *(optional)* The maximum number of connection failures to retry on. + * **timeout** - *(optional)* The timeout configuration to use when sending + requests. * **pool_limits** - *(optional)* The connection pool configuration to use when determining the maximum number of concurrently open HTTP connections. * **max_redirects** - *(optional)* The maximum number of redirect responses @@ -569,6 +569,7 @@ def request( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: request = self.build_request( @@ -582,7 +583,11 @@ def request( cookies=cookies, ) return self.send( - request, auth=auth, allow_redirects=allow_redirects, timeout=timeout, + request, + auth=auth, + allow_redirects=allow_redirects, + retries=retries, + timeout=timeout, ) def send( @@ -592,6 +597,7 @@ def send( stream: bool = False, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: if request.url.scheme not in ("http", "https"): @@ -599,7 +605,7 @@ def send( timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) - retries = self.retries + retries = self.retries if isinstance(retries, UnsetType) else Retries(retries) auth = self.build_auth(request, auth) @@ -758,6 +764,7 @@ def get( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return self.request( @@ -768,6 +775,7 @@ def get( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -780,6 +788,7 @@ def options( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return self.request( @@ -790,6 +799,7 @@ def options( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -802,6 +812,7 @@ def head( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = False, # NOTE: Differs to usual default. + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return self.request( @@ -812,6 +823,7 @@ def head( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -827,6 +839,7 @@ def post( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return self.request( @@ -840,6 +853,7 @@ def post( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -855,6 +869,7 @@ def put( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return self.request( @@ -868,6 +883,7 @@ def put( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -883,6 +899,7 @@ def patch( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return self.request( @@ -896,6 +913,7 @@ def patch( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -908,6 +926,7 @@ def delete( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return self.request( @@ -918,6 +937,7 @@ def delete( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -969,6 +989,8 @@ class AsyncClient(BaseClient): enabled. Defaults to `False`. * **proxies** - *(optional)* A dictionary mapping HTTP protocols to proxy URLs. + * **retries** - *(optional)* The maximum number of connection failures to + retry on. * **timeout** - *(optional)* The timeout configuration to use when sending requests. * **pool_limits** - *(optional)* The connection pool configuration to use @@ -1137,6 +1159,7 @@ async def request( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: request = self.build_request( @@ -1150,7 +1173,11 @@ async def request( cookies=cookies, ) response = await self.send( - request, auth=auth, allow_redirects=allow_redirects, timeout=timeout, + request, + auth=auth, + allow_redirects=allow_redirects, + retries=retries, + timeout=timeout, ) return response @@ -1161,6 +1188,7 @@ async def send( stream: bool = False, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: if request.url.scheme not in ("http", "https"): @@ -1168,7 +1196,7 @@ async def send( timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) - retries = self.retries + retries = self.retries if isinstance(retries, UnsetType) else Retries(retries) auth = self.build_auth(request, auth) @@ -1330,6 +1358,7 @@ async def get( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return await self.request( @@ -1340,6 +1369,7 @@ async def get( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -1352,6 +1382,7 @@ async def options( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return await self.request( @@ -1362,6 +1393,7 @@ async def options( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -1374,6 +1406,7 @@ async def head( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = False, # NOTE: Differs to usual default. + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return await self.request( @@ -1384,6 +1417,7 @@ async def head( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -1399,6 +1433,7 @@ async def post( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return await self.request( @@ -1412,6 +1447,7 @@ async def post( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -1427,6 +1463,7 @@ async def put( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return await self.request( @@ -1440,6 +1477,7 @@ async def put( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -1455,6 +1493,7 @@ async def patch( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return await self.request( @@ -1468,6 +1507,7 @@ async def patch( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -1480,6 +1520,7 @@ async def delete( cookies: CookieTypes = None, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, ) -> Response: return await self.request( @@ -1490,6 +1531,7 @@ async def delete( cookies=cookies, auth=auth, allow_redirects=allow_redirects, + retries=retries, timeout=timeout, ) @@ -1516,6 +1558,7 @@ def __init__( *, auth: AuthTypes = None, allow_redirects: bool = True, + retries: typing.Union[RetriesTypes, UnsetType] = UNSET, timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, close_client: bool = False, ) -> None: @@ -1523,6 +1566,7 @@ def __init__( self.request = request self.auth = auth self.allow_redirects = allow_redirects + self.retries = retries self.timeout = timeout self.close_client = close_client @@ -1532,6 +1576,7 @@ def __enter__(self) -> "Response": request=self.request, auth=self.auth, allow_redirects=self.allow_redirects, + retries=self.retries, timeout=self.timeout, stream=True, ) @@ -1554,6 +1599,7 @@ async def __aenter__(self) -> "Response": request=self.request, auth=self.auth, allow_redirects=self.allow_redirects, + retries=self.retries, timeout=self.timeout, stream=True, ) diff --git a/httpx/config.py b/httpx/config.py index 401a5dfded..4ddb1685d1 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -372,9 +372,7 @@ class Retries: ("HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE") ) - def __init__( - self, retries: RetriesTypes = 0, *, backoff_factor: float = None - ) -> None: + def __init__(self, retries: RetriesTypes, *, backoff_factor: float = None) -> None: if isinstance(retries, int): limit = retries else: diff --git a/tests/client/test_retries.py b/tests/client/test_retries.py index 2b6809e09e..1d2bc01d87 100644 --- a/tests/client/test_retries.py +++ b/tests/client/test_retries.py @@ -11,7 +11,7 @@ def test_retries_config() -> None: client = httpx.AsyncClient() - assert client.retries == httpx.Retries() == httpx.Retries(0) + assert client.retries == httpx.Retries(0) assert client.retries.limit == 0 client = httpx.AsyncClient(retries=3) @@ -27,8 +27,8 @@ def test_retries_config() -> None: @pytest.mark.parametrize( "retries, delays", [ - (httpx.Retries(), [0, 0.2, 0.4, 0.8, 1.6]), - (httpx.Retries(backoff_factor=0.1), [0, 0.1, 0.2, 0.4, 0.8]), + (httpx.Retries(3), [0, 0.2, 0.4, 0.8, 1.6]), + (httpx.Retries(3, backoff_factor=0.1), [0, 0.1, 0.2, 0.4, 0.8]), ], ) def test_retries_delays_sequence( diff --git a/tests/test_timeouts.py b/tests/test_timeouts.py index 8754449231..e394e0e301 100644 --- a/tests/test_timeouts.py +++ b/tests/test_timeouts.py @@ -26,7 +26,7 @@ async def test_write_timeout(server): async def test_connect_timeout(server): timeout = httpx.Timeout(connect_timeout=1e-6) - async with httpx.AsyncClient(timeout=timeout, retries=0) as client: + async with httpx.AsyncClient(timeout=timeout) as client: with pytest.raises(httpx.ConnectTimeout): # See https://stackoverflow.com/questions/100841/ await client.get("http://10.255.255.1/") @@ -37,9 +37,7 @@ async def test_pool_timeout(server): pool_limits = httpx.PoolLimits(hard_limit=1) timeout = httpx.Timeout(pool_timeout=1e-4) - async with httpx.AsyncClient( - pool_limits=pool_limits, timeout=timeout, retries=0 - ) as client: + async with httpx.AsyncClient(pool_limits=pool_limits, timeout=timeout) as client: async with client.stream("GET", server.url): with pytest.raises(httpx.PoolTimeout): await client.get("http://localhost:8000/") From a3e691781bdde4bc97ad8ba5720ea068b717bc9f Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Wed, 22 Jan 2020 08:32:11 -0500 Subject: [PATCH 20/22] Refactor for...else construct --- httpx/config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/httpx/config.py b/httpx/config.py index 6c2b98d1cc..e0921bdd0a 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -399,10 +399,11 @@ def __eq__(self, other: typing.Any) -> bool: @classmethod def should_retry_on_exception(cls, exc: HTTPError) -> bool: - for exc_cls in cls._RETRYABLE_EXCEPTIONS: - if isinstance(exc, exc_cls): - break - else: + is_retryable_exception_class = any( + isinstance(exc, exc_cls) for exc_cls in cls._RETRYABLE_EXCEPTIONS + ) + + if not is_retryable_exception_class: return False assert exc.request is not None From d3609c314344cec5efcf02eb6e21f540c7607004 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Wed, 22 Jan 2020 08:51:21 -0500 Subject: [PATCH 21/22] Drop DEFAULT_RETRIES_CONFIG for None, add disabling retries per-request --- docs/advanced.md | 6 ++++++ httpx/api.py | 19 +++++++++--------- httpx/client.py | 37 +++++++++++++++++++++++------------- httpx/config.py | 11 +++++++---- tests/client/test_retries.py | 13 ++++++++++++- 5 files changed, 58 insertions(+), 28 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index de5a9af3f3..ff192e4a54 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -511,6 +511,12 @@ Or enable them on a client instance, which results in the given `retries` being httpx.Client(retries=3) ``` +When using a client with retries enabled, you can still explicitly disable retries for a given request: + +```python +response = client.get("https://example.org", retries=None) +``` + ### Fine-tuning the retries configuration When enabling retries, the `retries` argument can also be an `httpx.Retries()` instance. It accepts the following arguments: diff --git a/httpx/api.py b/httpx/api.py index 74348575ba..50ab9ab5a6 100644 --- a/httpx/api.py +++ b/httpx/api.py @@ -3,7 +3,6 @@ from .auth import AuthTypes from .client import Client, StreamContextManager from .config import ( - DEFAULT_RETRIES_CONFIG, DEFAULT_TIMEOUT_CONFIG, CertTypes, RetriesTypes, @@ -33,7 +32,7 @@ def request( headers: HeaderTypes = None, cookies: CookieTypes = None, auth: AuthTypes = None, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, allow_redirects: bool = True, verify: VerifyTypes = True, @@ -118,7 +117,7 @@ def stream( headers: HeaderTypes = None, cookies: CookieTypes = None, auth: AuthTypes = None, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, allow_redirects: bool = True, verify: VerifyTypes = True, @@ -156,7 +155,7 @@ def get( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -194,7 +193,7 @@ def options( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -232,7 +231,7 @@ def head( allow_redirects: bool = False, # Note: Differs to usual default. cert: CertTypes = None, verify: VerifyTypes = True, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -275,7 +274,7 @@ def post( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -316,7 +315,7 @@ def put( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -357,7 +356,7 @@ def patch( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -395,7 +394,7 @@ def delete( allow_redirects: bool = True, cert: CertTypes = None, verify: VerifyTypes = True, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: diff --git a/httpx/client.py b/httpx/client.py index 63686c7dd6..6909a35fd5 100644 --- a/httpx/client.py +++ b/httpx/client.py @@ -10,7 +10,6 @@ from .config import ( DEFAULT_MAX_REDIRECTS, DEFAULT_POOL_LIMITS, - DEFAULT_RETRIES_CONFIG, DEFAULT_TIMEOUT_CONFIG, UNSET, CertTypes, @@ -68,8 +67,8 @@ def __init__( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, max_redirects: int = DEFAULT_MAX_REDIRECTS, base_url: URLTypes = None, trust_env: bool = True, @@ -86,8 +85,8 @@ def __init__( self._params = QueryParams(params) self._headers = Headers(headers) self._cookies = Cookies(cookies) - self.timeout = Timeout(timeout) self.retries = Retries(retries) + self.timeout = Timeout(timeout) self.max_redirects = max_redirects self.trust_env = trust_env self.netrc = NetRCInfo() @@ -453,8 +452,8 @@ def __init__( verify: VerifyTypes = True, cert: CertTypes = None, proxies: ProxiesTypes = None, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, pool_limits: PoolLimits = DEFAULT_POOL_LIMITS, max_redirects: int = DEFAULT_MAX_REDIRECTS, base_url: URLTypes = None, @@ -467,8 +466,8 @@ def __init__( params=params, headers=headers, cookies=cookies, - timeout=timeout, retries=retries, + timeout=timeout, max_redirects=max_redirects, base_url=base_url, trust_env=trust_env, @@ -633,6 +632,11 @@ def send_handling_retries( timeout: Timeout, allow_redirects: bool = True, ) -> Response: + if not retries.limit: + return self.send_handling_redirects( + request, auth=auth, timeout=timeout, allow_redirects=allow_redirects + ) + backend = self.backend retries_left = retries.limit delays = retries.get_delays() @@ -646,9 +650,10 @@ def send_handling_retries( allow_redirects=allow_redirects, ) except HTTPError as exc: - if not retries.limit or not retries.should_retry_on_exception(exc): - # We shouldn't retry at all in these cases, so let's re-raise - # immediately to avoid polluting logs or the exception stack. + if not retries.should_retry_on_exception(exc): + # Even if we have retries left, we're told to not even consider + # retrying in this case. So let's re-raise immediately to avoid + # polluting logs or the exception stack. raise logger.debug(f"HTTP Request failed: {exc!r}") @@ -1022,8 +1027,8 @@ def __init__( cert: CertTypes = None, http2: bool = False, proxies: ProxiesTypes = None, + retries: RetriesTypes = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, - retries: RetriesTypes = DEFAULT_RETRIES_CONFIG, pool_limits: PoolLimits = DEFAULT_POOL_LIMITS, max_redirects: int = DEFAULT_MAX_REDIRECTS, base_url: URLTypes = None, @@ -1038,8 +1043,8 @@ def __init__( params=params, headers=headers, cookies=cookies, - timeout=timeout, retries=retries, + timeout=timeout, max_redirects=max_redirects, base_url=base_url, trust_env=trust_env, @@ -1224,6 +1229,11 @@ async def send_handling_retries( timeout: Timeout, allow_redirects: bool = True, ) -> Response: + if not retries.limit: + return await self.send_handling_redirects( + request, auth=auth, timeout=timeout, allow_redirects=allow_redirects + ) + backend = lookup_backend() retries_left = retries.limit @@ -1238,9 +1248,10 @@ async def send_handling_retries( allow_redirects=allow_redirects, ) except HTTPError as exc: - if not retries.limit or not retries.should_retry_on_exception(exc): - # We shouldn't retry at all in these cases, so let's re-raise - # immediately to avoid polluting logs or the exception stack. + if not retries.should_retry_on_exception(exc): + # Even if we have retries left, we're told to not even consider + # retrying in this case. So let's re-raise immediately to avoid + # polluting logs or the exception stack. raise logger.debug(f"HTTP Request failed: {exc!r}") diff --git a/httpx/config.py b/httpx/config.py index e0921bdd0a..4f5c89e2bd 100644 --- a/httpx/config.py +++ b/httpx/config.py @@ -19,7 +19,7 @@ ProxiesTypes = typing.Union[ URLTypes, "Proxy", typing.Dict[URLTypes, typing.Union[URLTypes, "Proxy"]] ] -RetriesTypes = typing.Union[int, "Retries"] +RetriesTypes = typing.Union[None, int, "Retries"] DEFAULT_CIPHERS = ":".join( @@ -372,8 +372,12 @@ class Retries: ("HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE") ) - def __init__(self, retries: RetriesTypes, *, backoff_factor: float = None) -> None: - if isinstance(retries, int): + def __init__( + self, retries: RetriesTypes = None, *, backoff_factor: float = None + ) -> None: + if retries is None: + limit = 0 + elif isinstance(retries, int): limit = retries else: assert isinstance(retries, Retries) @@ -423,6 +427,5 @@ def get_delays(self) -> typing.Iterator[float]: DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0) -DEFAULT_RETRIES_CONFIG = Retries(0, backoff_factor=0.2) DEFAULT_POOL_LIMITS = PoolLimits(soft_limit=10, hard_limit=100) DEFAULT_MAX_REDIRECTS = 20 diff --git a/tests/client/test_retries.py b/tests/client/test_retries.py index 1d2bc01d87..9ca95ce2db 100644 --- a/tests/client/test_retries.py +++ b/tests/client/test_retries.py @@ -86,7 +86,7 @@ async def test_no_retries() -> None: @pytest.mark.usefixtures("async_environment") -async def test_retries() -> None: +async def test_client_retries() -> None: client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=3) response = await client.get("https://example.com") @@ -102,6 +102,17 @@ async def test_retries() -> None: assert response.status_code == 200 +@pytest.mark.usefixtures("async_environment") +async def test_request_retries() -> None: + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3)) + response = await client.get("https://example.com", retries=3) + assert response.status_code == 200 + + client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=3) + with pytest.raises(httpx.ConnectTimeout): + await client.get("https://example.com/connect_timeout", retries=None) + + @pytest.mark.usefixtures("async_environment") async def test_too_many_retries() -> None: client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=2), retries=1) From c435bde71127c9bbb56e1202118cfff5f45dad2e Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Thu, 23 Jan 2020 08:12:50 -0500 Subject: [PATCH 22/22] Tweak examples for client-level retries --- docs/advanced.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index ff192e4a54..8329cadfc0 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -508,13 +508,20 @@ Or enable them on a client instance, which results in the given `retries` being ```python # Retry at most 3 times on connection failures everywhere. -httpx.Client(retries=3) +with httpx.Client(retries=3) as client: + # This request now has retries enabled... + response = client.get("https://example.org") ``` -When using a client with retries enabled, you can still explicitly disable retries for a given request: +When using a client with retries enabled, you can still explicitly override or disable retries for a given request: ```python -response = client.get("https://example.org", retries=None) +with httpx.Client(retries=3) as client: + # Retry at most 5 times for this particular request. + response = client.get("https://example.org", retries=5) + + # Don't retry for this particular request. + response = client.get("https://example.org", retries=None) ``` ### Fine-tuning the retries configuration @@ -525,8 +532,9 @@ When enabling retries, the `retries` argument can also be an `httpx.Retries()` i * `backoff_factor` (optional), which defines the increase rate of the time to wait between retries. By default this is `0.2`, which corresponds to issuing a new request after `(0s, 0.2s, 0.4s, 0.8s, ...)`. (Note that most connection failures are immediately resolved by retrying, so HTTPX will always issue the initial retry right away.) ```python -# Retry at most 5 times on connection failures, +# Retry at most 5 times on connection failures everywhere, # and issue new requests after `(0s, 0.5s, 1s, 2s, 4s, ...)`. retries = httpx.Retries(5, backoff_factor=0.5) -client = httpx.Client(retries=retries) +with httpx.Client(retries=retries) as client: + ... ```